diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_21000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..c245ffe554066f7550379733c4c26d716732bb9a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 14.0, "train": {"Classification_binary": 0.24796493351459503, "Classification": 0.00045036073424853384, "Regression_pt": 0.3073813319206238, "Regression_eta": 0.0863073542714119, "Regression_sin_phi": 0.015295619145035744, "Regression_cos_phi": 0.016186676919460297, "Regression_energy": 0.36557886004447937, "Total": 3.8062331676483154, "Total_unweighted": 1.0391651365498547}, "train_group": {}, "valid": {"Classification_binary": 0.2518792152404785, "Classification": 0.0004192739337061842, "Regression_pt": 0.3154304424921672, "Regression_eta": 0.04695171117782593, "Regression_sin_phi": 0.011941542228062948, "Regression_cos_phi": 0.0118227352698644, "Regression_energy": 0.3295738299687703, "Total": 2.8064260482788086, "Total_unweighted": 0.9680187503108755, "metrics/particle/class_1/efficiency": 0.8901102542877197, "metrics/particle/class_1/pid_accuracy": 0.9881476163864136, "metrics/particle/class_2/efficiency": 0.05809731408953667, "metrics/particle/class_2/pid_accuracy": 0.0062500000931322575, "metrics/particle/class_3/efficiency": 0.3927185535430908, "metrics/particle/class_3/pid_accuracy": 0.9609609842300415, "metrics/particle/class_4/efficiency": 0.8095238208770752, "metrics/particle/class_4/pid_accuracy": 0.10084033757448196, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.29203540086746216, "metrics/particle/count/bias_mean": -21.41796875, "metrics/particle/count/mae": 21.44661521911621, "metrics/particle/count/prediction_mean": 58.1484375, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3156999945640564, "metrics/particle/event/energy_response_mean": 0.6868554949760437, "metrics/particle/event/met_abs_error": 15.092169761657715, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2678734064102173, "metrics/particle/event/scalar_pt_response_mean": 0.7359774708747864, "metrics/particle/event/vector_pt_closure": 0.11729150265455246, "metrics/particle/matched/delta_eta_abs_mean": 0.008352464064955711, "metrics/particle/matched/delta_phi_abs_mean": 0.0043267677538096905, "metrics/particle/matched/delta_r_mean": 0.010390879586338997, "metrics/particle/matched/energy_relative_abs_mean": 0.08189023286104202, "metrics/particle/matched/pid_accuracy": 0.9694981575012207, "metrics/particle/matched/pt_relative_abs_mean": 0.07332710176706314, "metrics/particle/matching/duplicate_fraction": 0.027117202058434486, "metrics/particle/matching/efficiency": 0.6384538412094116, "metrics/particle/matching/f1": 0.7377488017082214, "metrics/particle/matching/purity": 0.8736172914505005, "metrics/particle/matching/target_coverage_dr0p05": 0.6466362476348877, "metrics/particle/matching/target_coverage_dr0p10": 0.6660284399986267, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5655816793441772, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6554568409919739}, "step_train_time": 1.397691249847412, "step_valid_time": 9.687468443000398, "step_total_time": 5787.4764404296875, "elapsed_training_loop_seconds": 5787.4764404296875, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.794145926155678, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26579180755738374, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 584, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 509, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7334293948126801, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8715753424657534, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7965571205007824, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1284246575342466, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6671469740634006, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7928082191780822, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7245696400625978, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7334293948126801, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8554144230455383, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2606820952428066, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1456, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1315, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1206, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8550065019505851, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9031593406593407, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8784235136940548, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09684065934065934, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7841352405721717, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8282967032967034, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8056112224448898, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8550065019505851, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8253059028841327, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2967788157489476, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1080, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 963, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 849, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8359375, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8916666666666667, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8629032258064516, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10833333333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7369791666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7861111111111111, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.760752688172043, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8359375, "step_test_time": 21.229310315000475, "step_plot_time": 12.815548985000532, "step_validation_cycle_time": 44.20115943499968} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_22000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..8901ce8affe1de975c29b43a489571a895dcb976 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 14.666666666666666, "train": {"Classification_binary": 0.2503986954689026, "Classification": 0.00047194332000799477, "Regression_pt": 0.2631992697715759, "Regression_eta": 0.022652115672826767, "Regression_sin_phi": 0.005097675137221813, "Regression_cos_phi": 0.004393789451569319, "Regression_energy": 0.2623385787010193, "Total": 1.565084457397461, "Total_unweighted": 0.8085520675231237}, "train_group": {}, "valid": {"Classification_binary": 0.25095268090566, "Classification": 0.0004150123956302802, "Regression_pt": 0.3140642046928406, "Regression_eta": 0.04720162351926168, "Regression_sin_phi": 0.012088867525259653, "Regression_cos_phi": 0.011953444530566534, "Regression_energy": 0.3283235828081767, "Total": 2.822977383931478, "Total_unweighted": 0.9649994163773954, "metrics/particle/class_1/efficiency": 0.8922581076622009, "metrics/particle/class_1/pid_accuracy": 0.9887071847915649, "metrics/particle/class_2/efficiency": 0.05991285294294357, "metrics/particle/class_2/pid_accuracy": 0.0060606058686971664, "metrics/particle/class_3/efficiency": 0.39576199650764465, "metrics/particle/class_3/pid_accuracy": 0.9596270322799683, "metrics/particle/class_4/efficiency": 0.7925170063972473, "metrics/particle/class_4/pid_accuracy": 0.1373390555381775, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.12389380484819412, "metrics/particle/count/bias_mean": -21.21744728088379, "metrics/particle/count/mae": 21.23828125, "metrics/particle/count/prediction_mean": 58.34895706176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32076966762542725, "metrics/particle/event/energy_response_mean": 0.6849956512451172, "metrics/particle/event/met_abs_error": 15.319091796875, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26848509907722473, "metrics/particle/event/scalar_pt_response_mean": 0.7355186939239502, "metrics/particle/event/vector_pt_closure": 0.11826372146606445, "metrics/particle/matched/delta_eta_abs_mean": 0.00817905180156231, "metrics/particle/matched/delta_phi_abs_mean": 0.004337618593126535, "metrics/particle/matched/delta_r_mean": 0.010271136648952961, "metrics/particle/matched/energy_relative_abs_mean": 0.08278343826532364, "metrics/particle/matched/pid_accuracy": 0.9692814350128174, "metrics/particle/matched/pt_relative_abs_mean": 0.07493801414966583, "metrics/particle/matching/duplicate_fraction": 0.027314111590385437, "metrics/particle/matching/efficiency": 0.6408758163452148, "metrics/particle/matching/f1": 0.7394707202911377, "metrics/particle/matching/purity": 0.8739176988601685, "metrics/particle/matching/target_coverage_dr0p05": 0.6497782468795776, "metrics/particle/matching/target_coverage_dr0p10": 0.6686304211616516, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5654998421669006, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6572242379188538}, "step_train_time": 1.4080865383148193, "step_valid_time": 9.721324394000476, "step_total_time": 6065.151837825775, "elapsed_training_loop_seconds": 6065.151837825775, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8061174688857632, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26414963585578755, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 582, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 511, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7363112391930836, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8780068728522337, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8009404388714734, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12199312714776632, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6671469740634006, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7955326460481099, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.725705329153605, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7363112391930836, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8583091087282502, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2522981343883084, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1451, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1315, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1210, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8550065019505851, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9062715368711234, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8798929407828705, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09372846312887664, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7867360208062418, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.833907649896623, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8096353295416527, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8550065019505851, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.819501448408113, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.29528381075133736, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1081, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 849, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8324652777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8871415356151712, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8589341692789969, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11285846438482887, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7369791666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7853839037927844, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7604120017913122, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8324652777777778, "step_test_time": 21.38600082300036, "step_plot_time": 12.95955394199973, "step_validation_cycle_time": 44.48363208399951} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_23000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..8697ad0f834aa20eb104db93193008c0e3280ee5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 15.333333333333334, "train": {"Classification_binary": 0.2583095133304596, "Classification": 0.00034642338869161904, "Regression_pt": 0.33402568101882935, "Regression_eta": 0.042684897780418396, "Regression_sin_phi": 0.012091261334717274, "Regression_cos_phi": 0.006940957624465227, "Regression_energy": 0.33226826786994934, "Total": 2.4731507301330566, "Total_unweighted": 0.9866670023475308}, "train_group": {}, "valid": {"Classification_binary": 0.2504897514979045, "Classification": 0.00041094589202354353, "Regression_pt": 0.3077075481414795, "Regression_eta": 0.046626776456832886, "Regression_sin_phi": 0.011611297726631165, "Regression_cos_phi": 0.011799350380897522, "Regression_energy": 0.3213046391805013, "Total": 2.766855557759603, "Total_unweighted": 0.9499503092762704, "metrics/particle/class_1/efficiency": 0.8882150650024414, "metrics/particle/class_1/pid_accuracy": 0.9881934523582458, "metrics/particle/class_2/efficiency": 0.05628177151083946, "metrics/particle/class_2/pid_accuracy": 0.006451612804085016, "metrics/particle/class_3/efficiency": 0.4015445411205292, "metrics/particle/class_3/pid_accuracy": 0.9616295695304871, "metrics/particle/class_4/efficiency": 0.8129251599311829, "metrics/particle/class_4/pid_accuracy": 0.10041841119527817, "metrics/particle/class_5/efficiency": 0.9824561476707458, "metrics/particle/class_5/pid_accuracy": 0.2678571343421936, "metrics/particle/count/bias_mean": -21.5625, "metrics/particle/count/mae": 21.58333396911621, "metrics/particle/count/prediction_mean": 58.00390625, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3139006197452545, "metrics/particle/event/energy_response_mean": 0.6888015270233154, "metrics/particle/event/met_abs_error": 15.10596752166748, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2646605670452118, "metrics/particle/event/scalar_pt_response_mean": 0.739878237247467, "metrics/particle/event/vector_pt_closure": 0.11738339811563492, "metrics/particle/matched/delta_eta_abs_mean": 0.008239786140620708, "metrics/particle/matched/delta_phi_abs_mean": 0.004305145237594843, "metrics/particle/matched/delta_r_mean": 0.010292326100170612, "metrics/particle/matched/energy_relative_abs_mean": 0.08462970703840256, "metrics/particle/matched/pid_accuracy": 0.9696791768074036, "metrics/particle/matched/pt_relative_abs_mean": 0.07377939671278, "metrics/particle/matching/duplicate_fraction": 0.026039913296699524, "metrics/particle/matching/efficiency": 0.6411867737770081, "metrics/particle/matching/f1": 0.741685152053833, "metrics/particle/matching/purity": 0.8795429468154907, "metrics/particle/matching/target_coverage_dr0p05": 0.6480926871299744, "metrics/particle/matching/target_coverage_dr0p10": 0.6667157411575317, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5677582025527954, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6574532985687256}, "step_train_time": 1.4834656715393066, "step_valid_time": 9.725358855000195, "step_total_time": 6343.5153267383575, "elapsed_training_loop_seconds": 6343.5153267383575, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7997802784926182, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27220420121064925, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 587, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 512, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 464, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7377521613832853, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8722316865417377, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7993754879000781, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12776831345826234, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6685878962536023, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7904599659284497, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7244340359094458, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7377521613832853, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8693012484762807, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24809148189535402, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1452, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1328, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1224, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8634590377113134, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9146005509641874, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8882943143812709, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08539944903581267, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7958387516254877, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8429752066115702, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8187290969899665, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8634590377113134, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.827137856828898, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.291351031869143, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1079, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 951, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 848, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8255208333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.881371640407785, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8525324966382788, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11862835959221502, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7361111111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7859128822984245, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7601972209771403, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8255208333333334, "step_test_time": 21.04667240500021, "step_plot_time": 12.935911651000424, "step_validation_cycle_time": 44.17649511899981} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_24000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..f85f77d5c39783adc5f05d918b432f8312edd2d2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 16.0, "train": {"Classification_binary": 0.24952176213264465, "Classification": 0.0003704372502397746, "Regression_pt": 0.32329729199409485, "Regression_eta": 0.07389593869447708, "Regression_sin_phi": 0.013382569886744022, "Regression_cos_phi": 0.018262717872858047, "Regression_energy": 0.34246566891670227, "Total": 3.6223225593566895, "Total_unweighted": 1.0211963867477607}, "train_group": {}, "valid": {"Classification_binary": 0.2494985262552897, "Classification": 0.00040079668785134953, "Regression_pt": 0.30418725808461505, "Regression_eta": 0.04614782830079397, "Regression_sin_phi": 0.011931143701076508, "Regression_cos_phi": 0.01164837678273519, "Regression_energy": 0.31710028648376465, "Total": 2.7635342280069985, "Total_unweighted": 0.9409142162961265, "metrics/particle/class_1/efficiency": 0.8914052844047546, "metrics/particle/class_1/pid_accuracy": 0.9886609315872192, "metrics/particle/class_2/efficiency": 0.05809731408953667, "metrics/particle/class_2/pid_accuracy": 0.012500000186264515, "metrics/particle/class_3/efficiency": 0.38069695234298706, "metrics/particle/class_3/pid_accuracy": 0.9603277444839478, "metrics/particle/class_4/efficiency": 0.8231292366981506, "metrics/particle/class_4/pid_accuracy": 0.12396693974733353, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.28070175647735596, "metrics/particle/count/bias_mean": -22.25520896911621, "metrics/particle/count/mae": 22.27604103088379, "metrics/particle/count/prediction_mean": 57.31119918823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32789650559425354, "metrics/particle/event/energy_response_mean": 0.6740155220031738, "metrics/particle/event/met_abs_error": 14.832267761230469, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2772734463214874, "metrics/particle/event/scalar_pt_response_mean": 0.7262990474700928, "metrics/particle/event/vector_pt_closure": 0.11493057012557983, "metrics/particle/matched/delta_eta_abs_mean": 0.008657745085656643, "metrics/particle/matched/delta_phi_abs_mean": 0.004416583571583033, "metrics/particle/matched/delta_r_mean": 0.010732783935964108, "metrics/particle/matched/energy_relative_abs_mean": 0.08195366710424423, "metrics/particle/matched/pid_accuracy": 0.9698275923728943, "metrics/particle/matched/pt_relative_abs_mean": 0.07286122441291809, "metrics/particle/matching/duplicate_fraction": 0.025741225108504295, "metrics/particle/matching/efficiency": 0.6340354084968567, "metrics/particle/matching/f1": 0.737124502658844, "metrics/particle/matching/purity": 0.8802453875541687, "metrics/particle/matching/target_coverage_dr0p05": 0.6405976414680481, "metrics/particle/matching/target_coverage_dr0p10": 0.6609062552452087, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5606231689453125, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.649254560470581}, "step_train_time": 1.4427144527435303, "step_valid_time": 9.627448223000101, "step_total_time": 6620.498174190521, "elapsed_training_loop_seconds": 6620.498174190521, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7904215606322066, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27877353580709996, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 584, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 511, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 461, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7363112391930836, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.875, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7996870109546166, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.125, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6642651296829971, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7893835616438356, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7214397496087637, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7363112391930836, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8468536462925359, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24672812917039477, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1450, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1317, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1216, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8563068920676203, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9082758620689655, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8815261044176707, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09172413793103448, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7906371911573472, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8386206896551724, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8139223560910308, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8563068920676203, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8046649842763245, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2877182709545332, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1077, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 950, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 837, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8246527777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8820798514391829, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8524001794526693, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11792014856081709, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7265625, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7771587743732591, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7510094212651414, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8246527777777778, "step_test_time": 21.300130968000303, "step_plot_time": 12.804785668000477, "step_validation_cycle_time": 44.19782697099981} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_25000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..ffeedf609eb1d2c12d514c48fb5306487353e0bc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 16.666666666666668, "train": {"Classification_binary": 0.24292334914207458, "Classification": 0.0003634239546954632, "Regression_pt": 0.35660138726234436, "Regression_eta": 0.030243458226323128, "Regression_sin_phi": 0.012346460483968258, "Regression_cos_phi": 0.013610949739813805, "Regression_energy": 0.3340047597885132, "Total": 2.723153591156006, "Total_unweighted": 0.9900937885977328}, "train_group": {}, "valid": {"Classification_binary": 0.24923894802729288, "Classification": 0.0004039821991076072, "Regression_pt": 0.30441423257191974, "Regression_eta": 0.04687212904294332, "Regression_sin_phi": 0.011822811017433802, "Regression_cos_phi": 0.011786719163258871, "Regression_energy": 0.31940339008967084, "Total": 2.7763697306315103, "Total_unweighted": 0.943942212111627, "metrics/particle/class_1/efficiency": 0.8881834745407104, "metrics/particle/class_1/pid_accuracy": 0.9876595735549927, "metrics/particle/class_2/efficiency": 0.05954974517226219, "metrics/particle/class_2/pid_accuracy": 0.012195121496915817, "metrics/particle/class_3/efficiency": 0.39043596386909485, "metrics/particle/class_3/pid_accuracy": 0.9622917175292969, "metrics/particle/class_4/efficiency": 0.8231292366981506, "metrics/particle/class_4/pid_accuracy": 0.13636364042758942, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.40707963705062866, "metrics/particle/count/bias_mean": -22.13150978088379, "metrics/particle/count/mae": 22.14713478088379, "metrics/particle/count/prediction_mean": 57.43489456176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3218999207019806, "metrics/particle/event/energy_response_mean": 0.6802443861961365, "metrics/particle/event/met_abs_error": 15.214405059814453, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2744954526424408, "metrics/particle/event/scalar_pt_response_mean": 0.7292754054069519, "metrics/particle/event/vector_pt_closure": 0.11707072705030441, "metrics/particle/matched/delta_eta_abs_mean": 0.008261080831289291, "metrics/particle/matched/delta_phi_abs_mean": 0.004709992092102766, "metrics/particle/matched/delta_r_mean": 0.010569832287728786, "metrics/particle/matched/energy_relative_abs_mean": 0.08200760930776596, "metrics/particle/matched/pid_accuracy": 0.969872236251831, "metrics/particle/matched/pt_relative_abs_mean": 0.07250971347093582, "metrics/particle/matching/duplicate_fraction": 0.024144185706973076, "metrics/particle/matching/efficiency": 0.6366046667098999, "metrics/particle/matching/f1": 0.7394432425498962, "metrics/particle/matching/purity": 0.8819088935852051, "metrics/particle/matching/target_coverage_dr0p05": 0.643052339553833, "metrics/particle/matching/target_coverage_dr0p10": 0.6621500253677368, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5637161135673523, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6517583727836609}, "step_train_time": 1.5388131141662598, "step_valid_time": 9.639997762999883, "step_total_time": 6897.699452877045, "elapsed_training_loop_seconds": 6897.699452877045, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7918309453621263, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26546037273571643, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 573, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 501, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 451, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7219020172910663, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8743455497382199, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7908445146014207, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1256544502617801, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6498559077809798, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.787085514834206, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7119179163378059, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7219020172910663, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8527959939791655, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26468958743507964, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1451, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1318, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1210, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8569570871261378, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9083390764989663, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8819003011040482, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09166092350103378, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7867360208062418, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.833907649896623, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8096353295416527, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8569570871261378, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.817115758826644, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2831972205216202, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1079, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 948, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 840, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8229166666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8785912882298424, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8498431196772748, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12140871177015755, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7291666666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7784986098239111, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7530255490811295, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8229166666666666, "step_test_time": 21.44360460000007, "step_plot_time": 13.028635532999942, "step_validation_cycle_time": 44.5184836440003} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_26000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..efc8e744a5f01d58bdf97afcde244d3e26365170 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 17.333333333333332, "train": {"Classification_binary": 0.23835092782974243, "Classification": 0.0003761429979931563, "Regression_pt": 0.3064434826374054, "Regression_eta": 0.038808681070804596, "Regression_sin_phi": 0.01331343688070774, "Regression_cos_phi": 0.006639616098254919, "Regression_energy": 0.30540794134140015, "Total": 2.4374494552612305, "Total_unweighted": 0.9093402288563084}, "train_group": {}, "valid": {"Classification_binary": 0.24829012155532837, "Classification": 0.0004037234078471859, "Regression_pt": 0.2992990016937256, "Regression_eta": 0.045950278639793396, "Regression_sin_phi": 0.011673878878355026, "Regression_cos_phi": 0.011561164011557898, "Regression_energy": 0.312602420647939, "Total": 2.7336276372273765, "Total_unweighted": 0.9297805888345465, "metrics/particle/class_1/efficiency": 0.8922896981239319, "metrics/particle/class_1/pid_accuracy": 0.9884597659111023, "metrics/particle/class_2/efficiency": 0.06209150329232216, "metrics/particle/class_2/pid_accuracy": 0.04093567281961441, "metrics/particle/class_3/efficiency": 0.4174845814704895, "metrics/particle/class_3/pid_accuracy": 0.9658282995223999, "metrics/particle/class_4/efficiency": 0.8231292366981506, "metrics/particle/class_4/pid_accuracy": 0.09504131972789764, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.1150442510843277, "metrics/particle/count/bias_mean": -20.63411521911621, "metrics/particle/count/mae": 20.64713478088379, "metrics/particle/count/prediction_mean": 58.93229293823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3082902729511261, "metrics/particle/event/energy_response_mean": 0.6947038173675537, "metrics/particle/event/met_abs_error": 15.119532585144043, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25708943605422974, "metrics/particle/event/scalar_pt_response_mean": 0.7476778030395508, "metrics/particle/event/vector_pt_closure": 0.11673545837402344, "metrics/particle/matched/delta_eta_abs_mean": 0.008726044557988644, "metrics/particle/matched/delta_phi_abs_mean": 0.004669667221605778, "metrics/particle/matched/delta_r_mean": 0.01095920242369175, "metrics/particle/matched/energy_relative_abs_mean": 0.08396433293819427, "metrics/particle/matched/pid_accuracy": 0.9702131152153015, "metrics/particle/matched/pt_relative_abs_mean": 0.07378469407558441, "metrics/particle/matching/duplicate_fraction": 0.026358814910054207, "metrics/particle/matching/efficiency": 0.6504819393157959, "metrics/particle/matching/f1": 0.7473934292793274, "metrics/particle/matching/purity": 0.8782368302345276, "metrics/particle/matching/target_coverage_dr0p05": 0.6570114493370056, "metrics/particle/matching/target_coverage_dr0p10": 0.6774510145187378, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5742877125740051, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6673375964164734}, "step_train_time": 1.4769997596740723, "step_valid_time": 9.707817270000305, "step_total_time": 7176.054261684418, "elapsed_training_loop_seconds": 7176.054261684418, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8045737844428921, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26623513915553887, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 583, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 512, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 464, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7377521613832853, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8782161234991424, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8018794048551292, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12178387650085763, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6685878962536023, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7958833619210978, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7267032106499608, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7377521613832853, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8730833140592344, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2503464571995839, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1461, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1330, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1233, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8647594278283485, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9103353867214237, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8869623207735912, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08966461327857632, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8016905071521456, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8439425051334702, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8222740913637879, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8647594278283485, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8353038751266628, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.29274025056423136, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1085, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 965, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 872, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8376736111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8894009216589862, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8627626285203397, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11059907834101383, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7569444444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8036866359447005, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7796155565489495, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8376736111111112, "step_test_time": 21.055207395999787, "step_plot_time": 12.932689562000633, "step_validation_cycle_time": 44.175124626999605} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_27000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..fa4c7cf5bf1f1b5e2270bf8d04347ffbab66f878 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 18.0, "train": {"Classification_binary": 0.23012422025203705, "Classification": 0.0002795562904793769, "Regression_pt": 0.3334706127643585, "Regression_eta": 0.017697345465421677, "Regression_sin_phi": 0.01361779309809208, "Regression_cos_phi": 0.007936445064842701, "Regression_energy": 0.3234767019748688, "Total": 2.2512338161468506, "Total_unweighted": 0.9266026749101002}, "train_group": {}, "valid": {"Classification_binary": 0.24783921241760254, "Classification": 0.0003944524408628543, "Regression_pt": 0.2980796694755554, "Regression_eta": 0.047537813584009804, "Regression_sin_phi": 0.011858124285936356, "Regression_cos_phi": 0.011458127448956171, "Regression_energy": 0.3111452062924703, "Total": 2.7552353541056314, "Total_unweighted": 0.9283126059453934, "metrics/particle/class_1/efficiency": 0.8929214477539062, "metrics/particle/class_1/pid_accuracy": 0.987937331199646, "metrics/particle/class_2/efficiency": 0.06172839552164078, "metrics/particle/class_2/pid_accuracy": 0.0117647061124444, "metrics/particle/class_3/efficiency": 0.4147454798221588, "metrics/particle/class_3/pid_accuracy": 0.9648688435554504, "metrics/particle/class_4/efficiency": 0.8095238208770752, "metrics/particle/class_4/pid_accuracy": 0.13865546882152557, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.31578946113586426, "metrics/particle/count/bias_mean": -20.43359375, "metrics/particle/count/mae": 20.46224021911621, "metrics/particle/count/prediction_mean": 59.1328125, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31106480956077576, "metrics/particle/event/energy_response_mean": 0.6918454170227051, "metrics/particle/event/met_abs_error": 15.279112815856934, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2589052617549896, "metrics/particle/event/scalar_pt_response_mean": 0.7464367747306824, "metrics/particle/event/vector_pt_closure": 0.11743218451738358, "metrics/particle/matched/delta_eta_abs_mean": 0.007881621830165386, "metrics/particle/matched/delta_phi_abs_mean": 0.004503043368458748, "metrics/particle/matched/delta_r_mean": 0.010120383463799953, "metrics/particle/matched/energy_relative_abs_mean": 0.08414100855588913, "metrics/particle/matched/pid_accuracy": 0.9703978300094604, "metrics/particle/matched/pt_relative_abs_mean": 0.07494132965803146, "metrics/particle/matching/duplicate_fraction": 0.026709824800491333, "metrics/particle/matching/efficiency": 0.6495655179023743, "metrics/particle/matching/f1": 0.7452614903450012, "metrics/particle/matching/purity": 0.8740256428718567, "metrics/particle/matching/target_coverage_dr0p05": 0.661217212677002, "metrics/particle/matching/target_coverage_dr0p10": 0.6798075437545776, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5710965991020203, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6651284098625183}, "step_train_time": 1.4294142723083496, "step_valid_time": 9.689166669999395, "step_total_time": 7453.018676280975, "elapsed_training_loop_seconds": 7453.018676280975, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8041604309055465, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27682497479966106, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 509, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 462, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7334293948126801, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8700854700854701, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7959343236903831, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12991452991452992, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6657060518731989, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7897435897435897, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7224394057857702, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7334293948126801, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8771391533599473, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24822484450316173, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1457, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1330, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1224, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8647594278283485, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.91283459162663, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8881469115191987, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08716540837336993, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7958387516254877, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8400823610157858, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8173622704507513, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8647594278283485, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8307554448423591, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3030248544118308, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1086, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 962, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 860, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8350694444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8858195211786372, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8596961572832886, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1141804788213628, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7465277777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7918968692449355, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7685433422698839, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8350694444444444, "step_test_time": 21.453672172999177, "step_plot_time": 12.97874199800026, "step_validation_cycle_time": 44.539623445999496} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_28000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..321e46f944e1f2964373d321302a2d110df72fed --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 18.666666666666668, "train": {"Classification_binary": 0.24052664637565613, "Classification": 0.0003016450209543109, "Regression_pt": 0.25589558482170105, "Regression_eta": 0.02225782722234726, "Regression_sin_phi": 0.007210363168269396, "Regression_cos_phi": 0.013372309505939484, "Regression_energy": 0.267802894115448, "Total": 2.1634278297424316, "Total_unweighted": 0.8073672702303156}, "train_group": {}, "valid": {"Classification_binary": 0.24720837672551474, "Classification": 0.00039003925242771703, "Regression_pt": 0.2963283856709798, "Regression_eta": 0.04682912429173788, "Regression_sin_phi": 0.011497156073649725, "Regression_cos_phi": 0.011284074435631434, "Regression_energy": 0.30915600061416626, "Total": 2.709040323893229, "Total_unweighted": 0.9226931570641076, "metrics/particle/class_1/efficiency": 0.8986701965332031, "metrics/particle/class_1/pid_accuracy": 0.988119900226593, "metrics/particle/class_2/efficiency": 0.05882352963089943, "metrics/particle/class_2/pid_accuracy": 0.018518518656492233, "metrics/particle/class_3/efficiency": 0.41337594389915466, "metrics/particle/class_3/pid_accuracy": 0.9637401103973389, "metrics/particle/class_4/efficiency": 0.8503401279449463, "metrics/particle/class_4/pid_accuracy": 0.18000000715255737, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.28070175647735596, "metrics/particle/count/bias_mean": -20.86588478088379, "metrics/particle/count/mae": 20.89453125, "metrics/particle/count/prediction_mean": 58.70051956176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31365713477134705, "metrics/particle/event/energy_response_mean": 0.6896663308143616, "metrics/particle/event/met_abs_error": 15.165160179138184, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26502475142478943, "metrics/particle/event/scalar_pt_response_mean": 0.7400839924812317, "metrics/particle/event/vector_pt_closure": 0.11630050092935562, "metrics/particle/matched/delta_eta_abs_mean": 0.007881509140133858, "metrics/particle/matched/delta_phi_abs_mean": 0.004398901481181383, "metrics/particle/matched/delta_r_mean": 0.010026016272604465, "metrics/particle/matched/energy_relative_abs_mean": 0.08162283897399902, "metrics/particle/matched/pid_accuracy": 0.9704339504241943, "metrics/particle/matched/pt_relative_abs_mean": 0.07047120481729507, "metrics/particle/matching/duplicate_fraction": 0.025353800505399704, "metrics/particle/matching/efficiency": 0.6520202159881592, "metrics/particle/matching/f1": 0.7504166960716248, "metrics/particle/matching/purity": 0.8837895393371582, "metrics/particle/matching/target_coverage_dr0p05": 0.6585333943367004, "metrics/particle/matching/target_coverage_dr0p10": 0.6759945750236511, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5787389278411865, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6676812767982483}, "step_train_time": 1.4199244976043701, "step_valid_time": 9.623557747999257, "step_total_time": 7730.8285529613495, "elapsed_training_loop_seconds": 7730.8285529613495, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7985076904574893, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27740668149489134, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 510, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 457, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7348703170028819, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8717948717948718, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7974980453479281, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1282051282051282, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6585014409221902, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7811965811965812, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7146207974980453, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7348703170028819, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.864343990233434, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25408978379855696, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1451, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1318, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1211, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8569570871261378, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9083390764989663, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8819003011040482, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09166092350103378, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7873862158647594, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8345968297725707, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.810304449648712, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8569570871261378, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8197190112795245, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2835953997517118, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1087, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 957, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 855, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8307291666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8804047838086476, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8548459133541759, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11959521619135234, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7421875, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7865685372585096, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7637338097364895, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8307291666666666, "step_test_time": 21.230130445999748, "step_plot_time": 12.995724004999829, "step_validation_cycle_time": 44.32450776499991} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_29000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..33c713c228b069dd67a08896e1978b3f125aad26 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 19.333333333333332, "train": {"Classification_binary": 0.2279723435640335, "Classification": 0.00036340279621072114, "Regression_pt": 0.29064300656318665, "Regression_eta": 0.021095680072903633, "Regression_sin_phi": 0.010264307260513306, "Regression_cos_phi": 0.012571906670928001, "Regression_energy": 0.28103119134902954, "Total": 2.326197862625122, "Total_unweighted": 0.8439418382768054}, "train_group": {}, "valid": {"Classification_binary": 0.2470409075419108, "Classification": 0.0003896695173655947, "Regression_pt": 0.2927583654721578, "Regression_eta": 0.045771002769470215, "Regression_sin_phi": 0.011892547210057577, "Regression_cos_phi": 0.011675087114175161, "Regression_energy": 0.30511756738026935, "Total": 2.7383159001668296, "Total_unweighted": 0.9146451470054064, "metrics/particle/class_1/efficiency": 0.8951008915901184, "metrics/particle/class_1/pid_accuracy": 0.9901545643806458, "metrics/particle/class_2/efficiency": 0.05846042186021805, "metrics/particle/class_2/pid_accuracy": 0.018633540719747543, "metrics/particle/class_3/efficiency": 0.3970935046672821, "metrics/particle/class_3/pid_accuracy": 0.9642651677131653, "metrics/particle/class_4/efficiency": 0.8435373902320862, "metrics/particle/class_4/pid_accuracy": 0.12096773833036423, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.1150442510843277, "metrics/particle/count/bias_mean": -21.79557228088379, "metrics/particle/count/mae": 21.81119728088379, "metrics/particle/count/prediction_mean": 57.77083206176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32511621713638306, "metrics/particle/event/energy_response_mean": 0.6762032508850098, "metrics/particle/event/met_abs_error": 14.717872619628906, "metrics/particle/event/scalar_pt_relative_abs_error": 0.27469736337661743, "metrics/particle/event/scalar_pt_response_mean": 0.7286487221717834, "metrics/particle/event/vector_pt_closure": 0.11482399702072144, "metrics/particle/matched/delta_eta_abs_mean": 0.008012376725673676, "metrics/particle/matched/delta_phi_abs_mean": 0.0044814771972596645, "metrics/particle/matched/delta_r_mean": 0.010204923339188099, "metrics/particle/matched/energy_relative_abs_mean": 0.07861723005771637, "metrics/particle/matched/pid_accuracy": 0.9712962508201599, "metrics/particle/matched/pt_relative_abs_mean": 0.06974077969789505, "metrics/particle/matching/duplicate_fraction": 0.0231923907995224, "metrics/particle/matching/efficiency": 0.6431014537811279, "metrics/particle/matching/f1": 0.7451623678207397, "metrics/particle/matching/purity": 0.8857284784317017, "metrics/particle/matching/target_coverage_dr0p05": 0.6501709818840027, "metrics/particle/matching/target_coverage_dr0p10": 0.6679922342300415, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5703601837158203, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6580424308776855}, "step_train_time": 1.482508897781372, "step_valid_time": 9.677025319000677, "step_total_time": 8007.222347021103, "elapsed_training_loop_seconds": 8007.222347021103, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7945606821936329, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27682554025930783, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 581, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 505, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 457, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7276657060518732, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8691910499139415, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.792156862745098, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13080895008605853, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6585014409221902, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7865748709122203, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7168627450980393, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7276657060518732, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.854951111309811, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2563870992101429, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1455, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1323, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1218, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8602080624187256, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9092783505154639, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.884062813230872, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09072164948453608, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7919375812743823, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8371134020618557, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8138990978950885, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8602080624187256, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8124182600098926, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28088670502625723, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1073, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 944, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 840, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8194444444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8797763280521901, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8485393258426966, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12022367194780988, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7291666666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.782851817334576, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.755056179775281, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8194444444444444, "step_test_time": 21.154139887999918, "step_plot_time": 12.900920254000084, "step_validation_cycle_time": 44.13356725199992} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_3000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..1f6698b3d3acfad5ae147c6467c95266f1c4d880 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 2.0, "train": {"Classification_binary": 0.25323450565338135, "Classification": 0.0004651190247386694, "Regression_pt": 0.4705200791358948, "Regression_eta": 0.21395477652549744, "Regression_sin_phi": 0.014386644586920738, "Regression_cos_phi": 0.012107567861676216, "Regression_energy": 0.543748140335083, "Total": 5.3811354637146, "Total_unweighted": 1.5084168331231922}, "train_group": {}, "valid": {"Classification_binary": 0.28384031852086383, "Classification": 0.0005582295901452502, "Regression_pt": 0.4329607884089152, "Regression_eta": 0.0642855316400528, "Regression_sin_phi": 0.014341802646716436, "Regression_cos_phi": 0.015113968402147293, "Regression_energy": 0.4473681052525838, "Total": 3.586146672566732, "Total_unweighted": 1.2584687444614246, "metrics/particle/class_1/efficiency": 0.873306155204773, "metrics/particle/class_1/pid_accuracy": 0.9874855279922485, "metrics/particle/class_2/efficiency": 0.05156136676669121, "metrics/particle/class_2/pid_accuracy": 0.0, "metrics/particle/class_3/efficiency": 0.2674807906150818, "metrics/particle/class_3/pid_accuracy": 0.9526383280754089, "metrics/particle/class_4/efficiency": 0.8639456033706665, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.10526315867900848, "metrics/particle/count/bias_mean": -26.85025978088379, "metrics/particle/count/mae": 26.88150978088379, "metrics/particle/count/prediction_mean": 52.71614456176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.33180326223373413, "metrics/particle/event/energy_response_mean": 0.6709572672843933, "metrics/particle/event/met_abs_error": 15.478157043457031, "metrics/particle/event/scalar_pt_relative_abs_error": 0.287268728017807, "metrics/particle/event/scalar_pt_response_mean": 0.7166121602058411, "metrics/particle/event/vector_pt_closure": 0.11707965284585953, "metrics/particle/matched/delta_eta_abs_mean": 0.009377789683640003, "metrics/particle/matched/delta_phi_abs_mean": 0.0038108895532786846, "metrics/particle/matched/delta_r_mean": 0.010980769991874695, "metrics/particle/matched/energy_relative_abs_mean": 0.08134587109088898, "metrics/particle/matched/pid_accuracy": 0.9665520191192627, "metrics/particle/matched/pt_relative_abs_mean": 0.06629163771867752, "metrics/particle/matching/duplicate_fraction": 0.0186731219291687, "metrics/particle/matching/efficiency": 0.5758587121963501, "metrics/particle/matching/f1": 0.6927445530891418, "metrics/particle/matching/purity": 0.8691646456718445, "metrics/particle/matching/target_coverage_dr0p05": 0.5753186941146851, "metrics/particle/matching/target_coverage_dr0p10": 0.5964619517326355, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5258317589759827, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5948254466056824}, "step_train_time": 1.267988920211792, "step_valid_time": 9.663887217000138, "step_total_time": 799.6290905475616, "elapsed_training_loop_seconds": 799.6290905475616, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7865536415431497, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2920794831934037, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 595, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 512, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 449, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7377521613832853, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8605042016806723, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7944142746314973, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13949579831932774, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6469740634005764, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7546218487394958, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6966640806826998, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7377521613832853, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8274455866405701, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2832433589881842, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1448, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1308, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1172, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8504551365409623, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9033149171270718, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8760884125920965, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09668508287292818, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7620286085825748, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8093922651933702, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7849966510381782, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8504551365409623, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7898877010007463, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28815883915052776, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1062, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 935, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 806, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8116319444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8804143126177024, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8446251129177959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11958568738229755, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6996527777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7589453860640302, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7280939476061428, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8116319444444444, "step_test_time": 20.4985520419998, "step_plot_time": 12.632936806000089, "step_validation_cycle_time": 43.22908714899995} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..30a50a913245ed3a205dcc79cf242a94746d97ea --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 20.0, "train": {"Classification_binary": 0.24343571066856384, "Classification": 0.0002795427280943841, "Regression_pt": 0.24312734603881836, "Regression_eta": 0.023922763764858246, "Regression_sin_phi": 0.007272412069141865, "Regression_cos_phi": 0.009684241376817226, "Regression_energy": 0.2483224868774414, "Total": 1.9499545097351074, "Total_unweighted": 0.7760445035237353}, "train_group": {}, "valid": {"Classification_binary": 0.24608200788497925, "Classification": 0.0003838941144446532, "Regression_pt": 0.2911136945088704, "Regression_eta": 0.04554012914498647, "Regression_sin_phi": 0.011885049442450205, "Regression_cos_phi": 0.01136124258240064, "Regression_energy": 0.30372228225072223, "Total": 2.712073008219401, "Total_unweighted": 0.910088299928854, "metrics/particle/class_1/efficiency": 0.9025869369506836, "metrics/particle/class_1/pid_accuracy": 0.9881714582443237, "metrics/particle/class_2/efficiency": 0.06245461106300354, "metrics/particle/class_2/pid_accuracy": 0.023255813866853714, "metrics/particle/class_3/efficiency": 0.41733241081237793, "metrics/particle/class_3/pid_accuracy": 0.9651777744293213, "metrics/particle/class_4/efficiency": 0.8435373902320862, "metrics/particle/class_4/pid_accuracy": 0.24596774578094482, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.2368421107530594, "metrics/particle/count/bias_mean": -20.05989646911621, "metrics/particle/count/mae": 20.1015625, "metrics/particle/count/prediction_mean": 59.50651168823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3148428499698639, "metrics/particle/event/energy_response_mean": 0.6884387135505676, "metrics/particle/event/met_abs_error": 14.9299955368042, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2618073523044586, "metrics/particle/event/scalar_pt_response_mean": 0.7429128289222717, "metrics/particle/event/vector_pt_closure": 0.11546692997217178, "metrics/particle/matched/delta_eta_abs_mean": 0.00812512543052435, "metrics/particle/matched/delta_phi_abs_mean": 0.0042667752131819725, "metrics/particle/matched/delta_r_mean": 0.01018364168703556, "metrics/particle/matched/energy_relative_abs_mean": 0.08128789067268372, "metrics/particle/matched/pid_accuracy": 0.9710072875022888, "metrics/particle/matched/pt_relative_abs_mean": 0.0721122995018959, "metrics/particle/matching/duplicate_fraction": 0.027111005038022995, "metrics/particle/matching/efficiency": 0.6558822989463806, "metrics/particle/matching/f1": 0.7504868507385254, "metrics/particle/matching/purity": 0.8769829869270325, "metrics/particle/matching/target_coverage_dr0p05": 0.6659793257713318, "metrics/particle/matching/target_coverage_dr0p10": 0.6841769218444824, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5790334939956665, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6711996793746948}, "step_train_time": 1.5552654266357422, "step_valid_time": 9.741075744999762, "step_total_time": 8284.657896280289, "elapsed_training_loop_seconds": 8284.657896280289, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7950510313338642, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2746462854919077, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 513, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 466, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.739193083573487, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8769230769230769, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8021892103205629, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12307692307692308, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6714697406340058, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7965811965811965, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.72869429241595, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.739193083573487, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8607404716697616, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2537175552861439, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1458, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1327, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1231, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8628088426527959, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9101508916323731, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8858477970627503, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08984910836762688, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8003901170351105, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8443072702331962, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.821762349799733, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8628088426527959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8218275261640687, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2959138585934944, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1086, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 966, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 864, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8385416666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8895027624309392, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8632707774798928, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11049723756906077, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.75, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7955801104972375, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7721179624664879, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8385416666666666, "step_test_time": 21.361921300000176, "step_plot_time": 12.866489602000001, "step_validation_cycle_time": 44.385047239000414} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_31000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..b74aacdb91e2d63e32abedd67d5971fc017c202d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 20.666666666666668, "train": {"Classification_binary": 0.2338966578245163, "Classification": 0.00038153838249854743, "Regression_pt": 0.32970520853996277, "Regression_eta": 0.0801454484462738, "Regression_sin_phi": 0.011577479541301727, "Regression_cos_phi": 0.008730839006602764, "Regression_energy": 0.3661653995513916, "Total": 3.0353875160217285, "Total_unweighted": 1.0306025712925475}, "train_group": {}, "valid": {"Classification_binary": 0.24630290269851685, "Classification": 0.00038327490134785574, "Regression_pt": 0.2930118242899577, "Regression_eta": 0.048703327775001526, "Regression_sin_phi": 0.011798767993847529, "Regression_cos_phi": 0.011638830105463663, "Regression_energy": 0.3076020081837972, "Total": 2.7675580978393555, "Total_unweighted": 0.9194409359479323, "metrics/particle/class_1/efficiency": 0.8895416855812073, "metrics/particle/class_1/pid_accuracy": 0.9884951114654541, "metrics/particle/class_2/efficiency": 0.05846042186021805, "metrics/particle/class_2/pid_accuracy": 0.018633540719747543, "metrics/particle/class_3/efficiency": 0.3923381268978119, "metrics/particle/class_3/pid_accuracy": 0.9663531184196472, "metrics/particle/class_4/efficiency": 0.8401360511779785, "metrics/particle/class_4/pid_accuracy": 0.2388664036989212, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.38053098320961, "metrics/particle/count/bias_mean": -22.44140625, "metrics/particle/count/mae": 22.44921875, "metrics/particle/count/prediction_mean": 57.125, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3234240710735321, "metrics/particle/event/energy_response_mean": 0.6781157851219177, "metrics/particle/event/met_abs_error": 14.89248275756836, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2739803194999695, "metrics/particle/event/scalar_pt_response_mean": 0.7290897965431213, "metrics/particle/event/vector_pt_closure": 0.11614193767309189, "metrics/particle/matched/delta_eta_abs_mean": 0.008361023850739002, "metrics/particle/matched/delta_phi_abs_mean": 0.004955528303980827, "metrics/particle/matched/delta_r_mean": 0.010832274332642555, "metrics/particle/matched/energy_relative_abs_mean": 0.08134306967258453, "metrics/particle/matched/pid_accuracy": 0.9721253514289856, "metrics/particle/matched/pt_relative_abs_mean": 0.07065261900424957, "metrics/particle/matching/duplicate_fraction": 0.02183625102043152, "metrics/particle/matching/efficiency": 0.6381592750549316, "metrics/particle/matching/f1": 0.7429295182228088, "metrics/particle/matching/purity": 0.8888584971427917, "metrics/particle/matching/target_coverage_dr0p05": 0.6427250504493713, "metrics/particle/matching/target_coverage_dr0p10": 0.6626573204994202, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5658435225486755, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6526420712471008}, "step_train_time": 1.5963633060455322, "step_valid_time": 9.753474828001345, "step_total_time": 8562.631597280502, "elapsed_training_loop_seconds": 8562.631597280502, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7845301610829157, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26867707265814555, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 584, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 515, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7420749279538905, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8818493150684932, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8059467918622848, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11815068493150685, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6671469740634006, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7928082191780822, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7245696400625978, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7420749279538905, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8599894290367284, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25898220745868006, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1456, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1321, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1224, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8589076723016905, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9072802197802198, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8824315297261189, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09271978021978022, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7958387516254877, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8406593406593407, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8176352705410822, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8589076723016905, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8056448759918464, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2904680196773498, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1077, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 947, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 842, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8220486111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8792943361188487, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8497083894122925, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12070566388115135, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7309027777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7818012999071495, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7554957379991027, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8220486111111112, "step_test_time": 21.39176441500058, "step_plot_time": 12.978856364999956, "step_validation_cycle_time": 44.54377284600014} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_32000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..35258604ae82887d0abc6bc81cce23a092307b5d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 21.333333333333332, "train": {"Classification_binary": 0.2299853265285492, "Classification": 0.00032959997770376503, "Regression_pt": 0.21800634264945984, "Regression_eta": 0.03325144946575165, "Regression_sin_phi": 0.008414948359131813, "Regression_cos_phi": 0.008108502253890038, "Regression_energy": 0.22606661915779114, "Total": 2.0221407413482666, "Total_unweighted": 0.7241627883922774}, "train_group": {}, "valid": {"Classification_binary": 0.24562346935272217, "Classification": 0.000379919889383018, "Regression_pt": 0.2870658040046692, "Regression_eta": 0.047987466057141624, "Regression_sin_phi": 0.011988018949826559, "Regression_cos_phi": 0.011434458196163177, "Regression_energy": 0.30003219842910767, "Total": 2.748485565185547, "Total_unweighted": 0.9045113348790135, "metrics/particle/class_1/efficiency": 0.8967118263244629, "metrics/particle/class_1/pid_accuracy": 0.9876008033752441, "metrics/particle/class_2/efficiency": 0.06390704214572906, "metrics/particle/class_2/pid_accuracy": 0.028409091755747795, "metrics/particle/class_3/efficiency": 0.4125770330429077, "metrics/particle/class_3/pid_accuracy": 0.9645919799804688, "metrics/particle/class_4/efficiency": 0.8367347121238708, "metrics/particle/class_4/pid_accuracy": 0.2601625919342041, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.4385964870452881, "metrics/particle/count/bias_mean": -20.86067771911621, "metrics/particle/count/mae": 20.88150978088379, "metrics/particle/count/prediction_mean": 58.70573043823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3290363848209381, "metrics/particle/event/energy_response_mean": 0.6731047630310059, "metrics/particle/event/met_abs_error": 14.748490333557129, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2775743901729584, "metrics/particle/event/scalar_pt_response_mean": 0.7260782122612, "metrics/particle/event/vector_pt_closure": 0.11484711617231369, "metrics/particle/matched/delta_eta_abs_mean": 0.007993142120540142, "metrics/particle/matched/delta_phi_abs_mean": 0.004572316072881222, "metrics/particle/matched/delta_r_mean": 0.010256384499371052, "metrics/particle/matched/energy_relative_abs_mean": 0.07973175495862961, "metrics/particle/matched/pid_accuracy": 0.97100830078125, "metrics/particle/matched/pt_relative_abs_mean": 0.07024458050727844, "metrics/particle/matching/duplicate_fraction": 0.024730514734983444, "metrics/particle/matching/efficiency": 0.6508256196975708, "metrics/particle/matching/f1": 0.7490136027336121, "metrics/particle/matching/purity": 0.8820919990539551, "metrics/particle/matching/target_coverage_dr0p05": 0.6576824188232422, "metrics/particle/matching/target_coverage_dr0p10": 0.676190972328186, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5772006511688232, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6661921143531799}, "step_train_time": 1.6200518608093262, "step_valid_time": 9.738414872999783, "step_total_time": 8840.476172924042, "elapsed_training_loop_seconds": 8840.476172924042, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7879908197158025, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2676989940374238, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 507, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 456, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7305475504322767, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8666666666666667, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7928068803752932, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13333333333333333, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6570605187319885, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7794871794871795, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7130570758405004, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7305475504322767, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8486068811070608, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2602716027812315, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1450, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1318, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1212, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8569570871261378, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9089655172413793, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8821954484605087, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0910344827586207, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.788036410923277, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8358620689655173, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8112449799196787, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8569570871261378, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8057382083721066, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27722451022047356, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1079, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 952, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 845, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8263888888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8822984244670992, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8534289556252801, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11770157553290084, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7335069444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7831325301204819, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7575078440161362, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8263888888888888, "step_test_time": 21.405750284999158, "step_plot_time": 12.835856093000984, "step_validation_cycle_time": 44.387816550000935} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_33000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..9d6126da04727ce5540b428f36456712918a6ed7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 22.0, "train": {"Classification_binary": 0.23877504467964172, "Classification": 0.0003821218269877136, "Regression_pt": 0.3052036762237549, "Regression_eta": 0.05512119457125664, "Regression_sin_phi": 0.00934500340372324, "Regression_cos_phi": 0.011762545444071293, "Regression_energy": 0.3126694858074188, "Total": 2.7063424587249756, "Total_unweighted": 0.9332590719568543}, "train_group": {}, "valid": {"Classification_binary": 0.24453401565551758, "Classification": 0.00038104519868890446, "Regression_pt": 0.2834463318188985, "Regression_eta": 0.04451193908850352, "Regression_sin_phi": 0.011604214708010355, "Regression_cos_phi": 0.011501621454954147, "Regression_energy": 0.2956799864768982, "Total": 2.677583376566569, "Total_unweighted": 0.8916591544014711, "metrics/particle/class_1/efficiency": 0.9017341136932373, "metrics/particle/class_1/pid_accuracy": 0.9889659285545349, "metrics/particle/class_2/efficiency": 0.061365287750959396, "metrics/particle/class_2/pid_accuracy": 0.017751479521393776, "metrics/particle/class_3/efficiency": 0.4088868498802185, "metrics/particle/class_3/pid_accuracy": 0.9666914939880371, "metrics/particle/class_4/efficiency": 0.8469387888908386, "metrics/particle/class_4/pid_accuracy": 0.2610441744327545, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.24561403691768646, "metrics/particle/count/bias_mean": -20.76692771911621, "metrics/particle/count/mae": 20.81119728088379, "metrics/particle/count/prediction_mean": 58.79948043823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31720685958862305, "metrics/particle/event/energy_response_mean": 0.6859832406044006, "metrics/particle/event/met_abs_error": 14.365470886230469, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2659032642841339, "metrics/particle/event/scalar_pt_response_mean": 0.7373785972595215, "metrics/particle/event/vector_pt_closure": 0.11399681121110916, "metrics/particle/matched/delta_eta_abs_mean": 0.007880198769271374, "metrics/particle/matched/delta_phi_abs_mean": 0.004169044550508261, "metrics/particle/matched/delta_r_mean": 0.009902821853756905, "metrics/particle/matched/energy_relative_abs_mean": 0.07854465395212173, "metrics/particle/matched/pid_accuracy": 0.9721552729606628, "metrics/particle/matched/pt_relative_abs_mean": 0.06911257654428482, "metrics/particle/matching/duplicate_fraction": 0.02420390583574772, "metrics/particle/matching/efficiency": 0.6517747640609741, "metrics/particle/matching/f1": 0.7495977282524109, "metrics/particle/matching/purity": 0.8819699883460999, "metrics/particle/matching/target_coverage_dr0p05": 0.6603171229362488, "metrics/particle/matching/target_coverage_dr0p10": 0.6779583096504211, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5772170424461365, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6662248373031616}, "step_train_time": 1.5566647052764893, "step_valid_time": 9.712710613001036, "step_total_time": 9117.573209047318, "elapsed_training_loop_seconds": 9117.573209047318, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7895821284002628, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2699199261276174, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 586, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 513, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 464, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.739193083573487, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8754266211604096, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8015625, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12457337883959044, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6685878962536023, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7918088737201365, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.725, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.739193083573487, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.854716024735912, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26181250106119003, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1453, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1331, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1228, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8654096228868661, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9160357880247764, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8900033433634236, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08396421197522368, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7984395318595578, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8451479697178252, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8211300568371782, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8654096228868661, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.811634080914706, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.29170318017395114, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1080, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 956, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 860, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8298611111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8851851851851852, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8566308243727598, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11481481481481481, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7465277777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7962962962962963, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7706093189964157, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8298611111111112, "step_test_time": 21.573351156999706, "step_plot_time": 12.981288840999696, "step_validation_cycle_time": 44.756008330999975} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_34000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..de3b892cf37afeb37b9a620c09258be0e0a8965a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 22.666666666666668, "train": {"Classification_binary": 0.2605549991130829, "Classification": 0.0004227769677527249, "Regression_pt": 0.31020835041999817, "Regression_eta": 0.023729173466563225, "Regression_sin_phi": 0.006345899775624275, "Regression_cos_phi": 0.0069398535415530205, "Regression_energy": 0.3088586628437042, "Total": 1.8600207567214966, "Total_unweighted": 0.9170597161282785}, "train_group": {}, "valid": {"Classification_binary": 0.24513824780782065, "Classification": 0.0003756090300157666, "Regression_pt": 0.28636393944422406, "Regression_eta": 0.047246918082237244, "Regression_sin_phi": 0.011173648138840994, "Regression_cos_phi": 0.011260279764731726, "Regression_energy": 0.3008163571357727, "Total": 2.6758104960123696, "Total_unweighted": 0.9023749994036432, "metrics/particle/class_1/efficiency": 0.8942796587944031, "metrics/particle/class_1/pid_accuracy": 0.9882735013961792, "metrics/particle/class_2/efficiency": 0.060275960713624954, "metrics/particle/class_2/pid_accuracy": 0.0060240961611270905, "metrics/particle/class_3/efficiency": 0.40709882974624634, "metrics/particle/class_3/pid_accuracy": 0.9651434421539307, "metrics/particle/class_4/efficiency": 0.8503401279449463, "metrics/particle/class_4/pid_accuracy": 0.2199999988079071, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.4035087823867798, "metrics/particle/count/bias_mean": -20.89583396911621, "metrics/particle/count/mae": 20.91145896911621, "metrics/particle/count/prediction_mean": 58.67057418823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3198966681957245, "metrics/particle/event/energy_response_mean": 0.6832641959190369, "metrics/particle/event/met_abs_error": 14.689330101013184, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2677490711212158, "metrics/particle/event/scalar_pt_response_mean": 0.7360889911651611, "metrics/particle/event/vector_pt_closure": 0.11606931686401367, "metrics/particle/matched/delta_eta_abs_mean": 0.008316712453961372, "metrics/particle/matched/delta_phi_abs_mean": 0.004638689570128918, "metrics/particle/matched/delta_r_mean": 0.010590244084596634, "metrics/particle/matched/energy_relative_abs_mean": 0.0815967544913292, "metrics/particle/matched/pid_accuracy": 0.9713476300239563, "metrics/particle/matched/pt_relative_abs_mean": 0.07016081362962723, "metrics/particle/matching/duplicate_fraction": 0.02720877155661583, "metrics/particle/matching/efficiency": 0.6471108198165894, "metrics/particle/matching/f1": 0.7449277639389038, "metrics/particle/matching/purity": 0.8775827288627625, "metrics/particle/matching/target_coverage_dr0p05": 0.6545567512512207, "metrics/particle/matching/target_coverage_dr0p10": 0.6738998889923096, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5712111592292786, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6624118089675903}, "step_train_time": 1.4488866329193115, "step_valid_time": 9.726774341999771, "step_total_time": 9396.341699361801, "elapsed_training_loop_seconds": 9396.341699361801, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.79641406581092, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2763215165962484, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 589, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 508, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 460, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7319884726224783, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8624787775891342, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7918939984411536, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13752122241086587, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6628242074927954, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7809847198641766, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7170693686671863, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7319884726224783, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8573582554071754, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25882988372620197, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1455, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1326, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1230, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8621586475942783, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.911340206185567, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8860674908118944, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.088659793814433, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7997399219765929, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.845360824742268, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.821917808219178, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8621586475942783, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.815893633092547, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2888003804176439, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 950, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 851, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8246527777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8812615955473099, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.852017937219731, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11873840445269017, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7387152777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7894248608534323, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7632286995515695, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8246527777777778, "step_test_time": 21.266486267999426, "step_plot_time": 13.083493407000788, "step_validation_cycle_time": 44.57197639200058} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_35000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..7993c0f87f1de7eb0fa2199de06171ef556d0b19 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 23.333333333333332, "train": {"Classification_binary": 0.26172390580177307, "Classification": 0.000286288297502324, "Regression_pt": 0.30452993512153625, "Regression_eta": 0.019590605050325394, "Regression_sin_phi": 0.006721334997564554, "Regression_cos_phi": 0.005331193096935749, "Regression_energy": 0.29355961084365845, "Total": 1.6871899366378784, "Total_unweighted": 0.8917428732092958}, "train_group": {}, "valid": {"Classification_binary": 0.24423726399739584, "Classification": 0.00037357164546847343, "Regression_pt": 0.27946192026138306, "Regression_eta": 0.04689428210258484, "Regression_sin_phi": 0.011243379364411036, "Regression_cos_phi": 0.01133554552992185, "Regression_energy": 0.29268274704615277, "Total": 2.6695915857950845, "Total_unweighted": 0.8862287099473178, "metrics/particle/class_1/efficiency": 0.899807333946228, "metrics/particle/class_1/pid_accuracy": 0.9865903854370117, "metrics/particle/class_2/efficiency": 0.06608569622039795, "metrics/particle/class_2/pid_accuracy": 0.010989011265337467, "metrics/particle/class_3/efficiency": 0.4222399890422821, "metrics/particle/class_3/pid_accuracy": 0.9659428596496582, "metrics/particle/class_4/efficiency": 0.8605442047119141, "metrics/particle/class_4/pid_accuracy": 0.24901185929775238, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.37719297409057617, "metrics/particle/count/bias_mean": -20.49349021911621, "metrics/particle/count/mae": 20.52474021911621, "metrics/particle/count/prediction_mean": 59.07291793823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31216147541999817, "metrics/particle/event/energy_response_mean": 0.6910859942436218, "metrics/particle/event/met_abs_error": 14.628846168518066, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26247140765190125, "metrics/particle/event/scalar_pt_response_mean": 0.7415503859519958, "metrics/particle/event/vector_pt_closure": 0.11440237611532211, "metrics/particle/matched/delta_eta_abs_mean": 0.007821667939424515, "metrics/particle/matched/delta_phi_abs_mean": 0.004150227177888155, "metrics/particle/matched/delta_r_mean": 0.009840906597673893, "metrics/particle/matched/energy_relative_abs_mean": 0.07982715219259262, "metrics/particle/matched/pid_accuracy": 0.9700759649276733, "metrics/particle/matched/pt_relative_abs_mean": 0.06903856992721558, "metrics/particle/matching/duplicate_fraction": 0.02607564814388752, "metrics/particle/matching/efficiency": 0.6567987203598022, "metrics/particle/matching/f1": 0.7538858652114868, "metrics/particle/matching/purity": 0.8846544027328491, "metrics/particle/matching/target_coverage_dr0p05": 0.6645064949989319, "metrics/particle/matching/target_coverage_dr0p10": 0.6826550364494324, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.581030011177063, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6712487936019897}, "step_train_time": 1.5374760627746582, "step_valid_time": 9.654408076999971, "step_total_time": 9673.817191123962, "elapsed_training_loop_seconds": 9673.817191123962, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7937944271238824, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2624921806194831, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 589, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 508, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 460, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7319884726224783, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8624787775891342, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7918939984411536, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13752122241086587, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6628242074927954, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7809847198641766, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7170693686671863, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7319884726224783, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8695305587819818, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2516089035398763, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1455, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1323, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1229, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8602080624187256, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9092783505154639, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.884062813230872, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09072164948453608, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7990897269180754, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8446735395189003, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8212495823588373, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8602080624187256, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8212249200166553, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28082799459859087, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1079, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 965, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 868, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8376736111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8943466172381835, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8650829224562976, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1056533827618165, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7534722222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8044485634847081, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7781264007171672, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8376736111111112, "step_test_time": 21.49436013900049, "step_plot_time": 13.018994251999175, "step_validation_cycle_time": 44.66720465599974} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_36000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..d744f32be4abf94fb8d7a0627cd1b38a8fe7d5e0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 24.0, "train": {"Classification_binary": 0.22624729573726654, "Classification": 0.0003781919367611408, "Regression_pt": 0.32349011301994324, "Regression_eta": 0.017035329714417458, "Regression_sin_phi": 0.0071995980106294155, "Regression_cos_phi": 0.004850289318710566, "Regression_energy": 0.3083591163158417, "Total": 1.665322184562683, "Total_unweighted": 0.88755993405357}, "train_group": {}, "valid": {"Classification_binary": 0.2435738444328308, "Classification": 0.0003720535508667429, "Regression_pt": 0.2805091937383016, "Regression_eta": 0.045728142062822975, "Regression_sin_phi": 0.011228801061709722, "Regression_cos_phi": 0.011148234208424887, "Regression_energy": 0.29230721791585285, "Total": 2.6418498357137046, "Total_unweighted": 0.8848674869708095, "metrics/particle/class_1/efficiency": 0.901070773601532, "metrics/particle/class_1/pid_accuracy": 0.9875556230545044, "metrics/particle/class_2/efficiency": 0.06245461106300354, "metrics/particle/class_2/pid_accuracy": 0.01744186133146286, "metrics/particle/class_3/efficiency": 0.4329681098461151, "metrics/particle/class_3/pid_accuracy": 0.9715315103530884, "metrics/particle/class_4/efficiency": 0.8503401279449463, "metrics/particle/class_4/pid_accuracy": 0.18400000035762787, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.429824560880661, "metrics/particle/count/bias_mean": -19.69921875, "metrics/particle/count/mae": 19.75130271911621, "metrics/particle/count/prediction_mean": 59.8671875, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.30926308035850525, "metrics/particle/event/energy_response_mean": 0.692521870136261, "metrics/particle/event/met_abs_error": 14.623123168945312, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25586003065109253, "metrics/particle/event/scalar_pt_response_mean": 0.7474715113639832, "metrics/particle/event/vector_pt_closure": 0.11480437964200974, "metrics/particle/matched/delta_eta_abs_mean": 0.0076714614406228065, "metrics/particle/matched/delta_phi_abs_mean": 0.004281030967831612, "metrics/particle/matched/delta_r_mean": 0.009786308743059635, "metrics/particle/matched/energy_relative_abs_mean": 0.0801076665520668, "metrics/particle/matched/pid_accuracy": 0.972381591796875, "metrics/particle/matched/pt_relative_abs_mean": 0.06964019685983658, "metrics/particle/matching/duplicate_fraction": 0.02677367441356182, "metrics/particle/matching/efficiency": 0.6618554592132568, "metrics/particle/matching/f1": 0.7553625702857971, "metrics/particle/matching/purity": 0.8796380758285522, "metrics/particle/matching/target_coverage_dr0p05": 0.6710196733474731, "metrics/particle/matching/target_coverage_dr0p10": 0.6889063715934753, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5845648050308228, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6775655746459961}, "step_train_time": 1.467193603515625, "step_valid_time": 9.739965671000391, "step_total_time": 9952.681965827942, "elapsed_training_loop_seconds": 9952.681965827942, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8011099053556657, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28128716015820265, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 591, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 517, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 468, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7449567723342939, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8747884940778342, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8046692607003891, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12521150592216582, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6743515850144092, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7918781725888325, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7284046692607004, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7449567723342939, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8758003703374869, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25543348906808006, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1460, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1331, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1243, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8654096228868661, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9116438356164384, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8879252835223482, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08835616438356164, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8081924577373212, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8513698630136987, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.829219479653102, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8654096228868661, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8237089953944899, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28573544334387135, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1077, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 955, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 870, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8289930555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8867223769730733, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8568864961866308, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11327762302692665, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7552083333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.807799442896936, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7806191117092867, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8289930555555556, "step_test_time": 22.382350781999776, "step_plot_time": 12.978640067000015, "step_validation_cycle_time": 45.577847446000305} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_37000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..9c5632eb0ed09d16b1193a48d49bb8a8687ad36c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 24.666666666666668, "train": {"Classification_binary": 0.251930832862854, "Classification": 0.00036284560337662697, "Regression_pt": 0.2893894612789154, "Regression_eta": 0.058957427740097046, "Regression_sin_phi": 0.007023156154900789, "Regression_cos_phi": 0.011795436963438988, "Regression_energy": 0.30036595463752747, "Total": 2.603203296661377, "Total_unweighted": 0.9198251152411103}, "train_group": {}, "valid": {"Classification_binary": 0.24383548895517984, "Classification": 0.00037248785762737197, "Regression_pt": 0.27795203526814777, "Regression_eta": 0.045224045713742576, "Regression_sin_phi": 0.011380719641844431, "Regression_cos_phi": 0.011231801162163416, "Regression_energy": 0.2903197407722473, "Total": 2.647426128387451, "Total_unweighted": 0.8803163193709527, "metrics/particle/class_1/efficiency": 0.9041346907615662, "metrics/particle/class_1/pid_accuracy": 0.9868641495704651, "metrics/particle/class_2/efficiency": 0.06790123134851456, "metrics/particle/class_2/pid_accuracy": 0.03208556026220322, "metrics/particle/class_3/efficiency": 0.4345278739929199, "metrics/particle/class_3/pid_accuracy": 0.9683943390846252, "metrics/particle/class_4/efficiency": 0.8639456033706665, "metrics/particle/class_4/pid_accuracy": 0.29133859276771545, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.3947368562221527, "metrics/particle/count/bias_mean": -19.24869728088379, "metrics/particle/count/mae": 19.27474021911621, "metrics/particle/count/prediction_mean": 60.31770706176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3074372708797455, "metrics/particle/event/energy_response_mean": 0.6939143538475037, "metrics/particle/event/met_abs_error": 14.163959503173828, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25502943992614746, "metrics/particle/event/scalar_pt_response_mean": 0.7479548454284668, "metrics/particle/event/vector_pt_closure": 0.11304295063018799, "metrics/particle/matched/delta_eta_abs_mean": 0.007761052809655666, "metrics/particle/matched/delta_phi_abs_mean": 0.0044616940431296825, "metrics/particle/matched/delta_r_mean": 0.009982919320464134, "metrics/particle/matched/energy_relative_abs_mean": 0.0791318416595459, "metrics/particle/matched/pid_accuracy": 0.9712568521499634, "metrics/particle/matched/pt_relative_abs_mean": 0.07050079852342606, "metrics/particle/matching/duplicate_fraction": 0.027027027681469917, "metrics/particle/matching/efficiency": 0.6644247174263, "metrics/particle/matching/f1": 0.7558525800704956, "metrics/particle/matching/purity": 0.876457154750824, "metrics/particle/matching/target_coverage_dr0p05": 0.673916220664978, "metrics/particle/matching/target_coverage_dr0p10": 0.6926538944244385, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5847448110580444, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6798893809318542}, "step_train_time": 1.4283111095428467, "step_valid_time": 9.685696806998749, "step_total_time": 10231.09434223175, "elapsed_training_loop_seconds": 10231.09434223175, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8021452861029408, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2581620920333062, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 591, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 470, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7463976945244957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8764805414551607, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8062256809338522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12351945854483926, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6772334293948127, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7952622673434856, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7315175097276264, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7463976945244957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8732805888464449, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2487280102882029, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1459, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1339, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1245, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8706111833550065, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9177518848526388, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8935602268935602, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0822481151473612, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8094928478543563, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8533241946538725, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8308308308308309, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8706111833550065, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8252609949642038, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2942265464734437, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 958, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 875, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8315972222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8886827458256029, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8591928251121076, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11131725417439703, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7595486111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8116883116883117, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7847533632286996, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8315972222222222, "step_test_time": 21.21028067500083, "step_plot_time": 13.153731825999785, "step_validation_cycle_time": 44.52534375799951} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_38000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..24eb844befe4533e390724c006c67e745c4457e2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 25.333333333333332, "train": {"Classification_binary": 0.241084024310112, "Classification": 0.000371927279047668, "Regression_pt": 0.25323474407196045, "Regression_eta": 0.05633414164185524, "Regression_sin_phi": 0.007148693315684795, "Regression_cos_phi": 0.005635492503643036, "Regression_energy": 0.2672032415866852, "Total": 2.165276527404785, "Total_unweighted": 0.8310122647089884}, "train_group": {}, "valid": {"Classification_binary": 0.24328649044036865, "Classification": 0.0003667425674696763, "Regression_pt": 0.27884525060653687, "Regression_eta": 0.04575761159261068, "Regression_sin_phi": 0.01116406669219335, "Regression_cos_phi": 0.010973667105038961, "Regression_energy": 0.2909940878550212, "Total": 2.6244057019551597, "Total_unweighted": 0.8813879168592396, "metrics/particle/class_1/efficiency": 0.900881290435791, "metrics/particle/class_1/pid_accuracy": 0.9870621562004089, "metrics/particle/class_2/efficiency": 0.06390704214572906, "metrics/particle/class_2/pid_accuracy": 0.028409091755747795, "metrics/particle/class_3/efficiency": 0.42041391134262085, "metrics/particle/class_3/pid_accuracy": 0.9677857160568237, "metrics/particle/class_4/efficiency": 0.8605442047119141, "metrics/particle/class_4/pid_accuracy": 0.3122529685497284, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.48245614767074585, "metrics/particle/count/bias_mean": -20.63802146911621, "metrics/particle/count/mae": 20.67708396911621, "metrics/particle/count/prediction_mean": 58.92838668823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3059304654598236, "metrics/particle/event/energy_response_mean": 0.6982923150062561, "metrics/particle/event/met_abs_error": 14.571661949157715, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2584418058395386, "metrics/particle/event/scalar_pt_response_mean": 0.7462311387062073, "metrics/particle/event/vector_pt_closure": 0.11607208847999573, "metrics/particle/matched/delta_eta_abs_mean": 0.008436315692961216, "metrics/particle/matched/delta_phi_abs_mean": 0.004252483602613211, "metrics/particle/matched/delta_r_mean": 0.010427484288811684, "metrics/particle/matched/energy_relative_abs_mean": 0.0830659344792366, "metrics/particle/matched/pid_accuracy": 0.9718559384346008, "metrics/particle/matched/pt_relative_abs_mean": 0.06939162313938141, "metrics/particle/matching/duplicate_fraction": 0.026360563933849335, "metrics/particle/matching/efficiency": 0.6564714312553406, "metrics/particle/matching/f1": 0.7542965412139893, "metrics/particle/matching/purity": 0.8863822221755981, "metrics/particle/matching/target_coverage_dr0p05": 0.6636555790901184, "metrics/particle/matching/target_coverage_dr0p10": 0.6814931035041809, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5816681981086731, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6712979078292847}, "step_train_time": 1.4141857624053955, "step_valid_time": 9.633340086998942, "step_total_time": 10509.26925110817, "elapsed_training_loop_seconds": 10509.26925110817, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8013398566754002, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2757439870836428, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 591, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 511, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 458, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7363112391930836, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8646362098138748, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7953307392996108, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1353637901861252, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.659942363112392, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7749576988155669, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7128404669260701, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7363112391930836, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.86372182147293, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2548813404639587, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1459, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1327, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1235, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8628088426527959, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9095270733379027, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8855522188855522, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09047292666209733, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8029908972691807, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8464701850582591, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8241574908241575, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8628088426527959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8180783270310868, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.29101042522642384, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 954, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 858, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.828125, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8849721706864564, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8556053811659193, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1150278293135436, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7447916666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7959183673469388, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7695067264573991, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.828125, "step_test_time": 21.435063541999625, "step_plot_time": 13.008263685000202, "step_validation_cycle_time": 44.58588478800084} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_39000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_39000.json new file mode 100644 index 0000000000000000000000000000000000000000..9573ed6a8abb4cb38c50f15b9399825b35085603 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_39000.json @@ -0,0 +1 @@ +{"epoch": 26.0, "train": {"Classification_binary": 0.23659570515155792, "Classification": 0.00039912573993206024, "Regression_pt": 0.29495158791542053, "Regression_eta": 0.03906993195414543, "Regression_sin_phi": 0.00693066930398345, "Regression_cos_phi": 0.008805599994957447, "Regression_energy": 0.319725900888443, "Total": 2.172744035720825, "Total_unweighted": 0.9064785209484398}, "train_group": {}, "valid": {"Classification_binary": 0.2438443104426066, "Classification": 0.0003669274350007375, "Regression_pt": 0.2764423688252767, "Regression_eta": 0.044213185707728066, "Regression_sin_phi": 0.010688054064909617, "Regression_cos_phi": 0.010930174340804419, "Regression_energy": 0.2881689667701721, "Total": 2.5693252881368003, "Total_unweighted": 0.8746539875864981, "metrics/particle/class_1/efficiency": 0.8987017869949341, "metrics/particle/class_1/pid_accuracy": 0.9884366393089294, "metrics/particle/class_2/efficiency": 0.06790123134851456, "metrics/particle/class_2/pid_accuracy": 0.02139037474989891, "metrics/particle/class_3/efficiency": 0.4184737205505371, "metrics/particle/class_3/pid_accuracy": 0.9646363854408264, "metrics/particle/class_4/efficiency": 0.853741466999054, "metrics/particle/class_4/pid_accuracy": 0.26693227887153625, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.2631579041481018, "metrics/particle/count/bias_mean": -20.64583396911621, "metrics/particle/count/mae": 20.66927146911621, "metrics/particle/count/prediction_mean": 58.92057418823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3086734712123871, "metrics/particle/event/energy_response_mean": 0.6963109970092773, "metrics/particle/event/met_abs_error": 14.530109405517578, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2563192844390869, "metrics/particle/event/scalar_pt_response_mean": 0.7468697428703308, "metrics/particle/event/vector_pt_closure": 0.11406894773244858, "metrics/particle/matched/delta_eta_abs_mean": 0.008562165312469006, "metrics/particle/matched/delta_phi_abs_mean": 0.004410610534250736, "metrics/particle/matched/delta_r_mean": 0.010649111121892929, "metrics/particle/matched/energy_relative_abs_mean": 0.07946296781301498, "metrics/particle/matched/pid_accuracy": 0.9707779288291931, "metrics/particle/matched/pt_relative_abs_mean": 0.06961977481842041, "metrics/particle/matching/duplicate_fraction": 0.02556849643588066, "metrics/particle/matching/efficiency": 0.6546549201011658, "metrics/particle/matching/f1": 0.7522518038749695, "metrics/particle/matching/purity": 0.884046733379364, "metrics/particle/matching/target_coverage_dr0p05": 0.661462664604187, "metrics/particle/matching/target_coverage_dr0p10": 0.6807403564453125, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5785098075866699, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6695141196250916}, "step_train_time": 1.4550132751464844, "step_valid_time": 9.676539505999244, "step_total_time": 10786.61008810997, "elapsed_training_loop_seconds": 10786.61008810997, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8060231843132433, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28223244585294205, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 591, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 466, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7463976945244957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8764805414551607, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8062256809338522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12351945854483926, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6714697406340058, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7884940778341794, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7252918287937743, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7463976945244957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8673009151425295, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24643625957945348, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1459, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1338, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1245, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8699609882964889, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9170664838930774, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8928928928928929, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08293351610692255, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8094928478543563, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8533241946538725, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8308308308308309, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8699609882964889, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.819537258727718, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28596403610040955, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1079, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 962, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 861, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8350694444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.891566265060241, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8623935454952936, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10843373493975904, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7473958333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7979610750695088, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7718511878081578, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8350694444444444, "step_test_time": 21.479475189000368, "step_plot_time": 13.149519745000362, "step_validation_cycle_time": 44.810665140999845} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_4000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..530783646a8023c10459096f2594c8c5fc75bf34 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 2.6666666666666665, "train": {"Classification_binary": 0.27555617690086365, "Classification": 0.0005829792935401201, "Regression_pt": 0.4162198007106781, "Regression_eta": 0.07063140720129013, "Regression_sin_phi": 0.01203989703208208, "Regression_cos_phi": 0.010852145031094551, "Regression_energy": 0.43245500326156616, "Total": 3.2528772354125977, "Total_unweighted": 1.2183374094311148}, "train_group": {}, "valid": {"Classification_binary": 0.2778642574946086, "Classification": 0.0005331562521557013, "Regression_pt": 0.41204047203063965, "Regression_eta": 0.06138894955317179, "Regression_sin_phi": 0.014180928468704224, "Regression_cos_phi": 0.01454403375585874, "Regression_energy": 0.4255097309748332, "Total": 3.4665867487589517, "Total_unweighted": 1.2060615285299718, "metrics/particle/class_1/efficiency": 0.8759910464286804, "metrics/particle/class_1/pid_accuracy": 0.9868748188018799, "metrics/particle/class_2/efficiency": 0.054103121161460876, "metrics/particle/class_2/pid_accuracy": 0.00671140942722559, "metrics/particle/class_3/efficiency": 0.29498592019081116, "metrics/particle/class_3/pid_accuracy": 0.9488006234169006, "metrics/particle/class_4/efficiency": 0.853741466999054, "metrics/particle/class_4/pid_accuracy": 0.011952191591262817, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.12280701845884323, "metrics/particle/count/bias_mean": -25.26822853088379, "metrics/particle/count/mae": 25.3046875, "metrics/particle/count/prediction_mean": 54.29817581176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32337355613708496, "metrics/particle/event/energy_response_mean": 0.679004967212677, "metrics/particle/event/met_abs_error": 15.225208282470703, "metrics/particle/event/scalar_pt_relative_abs_error": 0.27957719564437866, "metrics/particle/event/scalar_pt_response_mean": 0.7244946360588074, "metrics/particle/event/vector_pt_closure": 0.11630219221115112, "metrics/particle/matched/delta_eta_abs_mean": 0.009243602864444256, "metrics/particle/matched/delta_phi_abs_mean": 0.0037988913245499134, "metrics/particle/matched/delta_r_mean": 0.010846692137420177, "metrics/particle/matched/energy_relative_abs_mean": 0.08301615715026855, "metrics/particle/matched/pid_accuracy": 0.9650843143463135, "metrics/particle/matched/pt_relative_abs_mean": 0.07110186666250229, "metrics/particle/matching/duplicate_fraction": 0.0217021182179451, "metrics/particle/matching/efficiency": 0.5891469120979309, "metrics/particle/matching/f1": 0.7003540396690369, "metrics/particle/matching/purity": 0.8633126020431519, "metrics/particle/matching/target_coverage_dr0p05": 0.5909633636474609, "metrics/particle/matching/target_coverage_dr0p10": 0.6131376028060913, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5329995155334473, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6085718274116516}, "step_train_time": 1.3220374584197998, "step_valid_time": 9.694642236999925, "step_total_time": 1076.6447494029999, "elapsed_training_loop_seconds": 1076.6447494029999, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7976063797265485, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28232668787610027, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 588, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 514, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 456, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7406340057636888, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8741496598639455, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8018720748829953, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12585034013605442, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6570605187319885, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7755102040816326, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7113884555382215, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7406340057636888, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8425833884116588, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2794779098231891, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1456, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1319, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1189, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8576072821846554, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9059065934065934, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8810955243820975, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09409340659340659, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7730819245773732, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8166208791208791, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7942551770207081, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8576072821846554, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7976905575469275, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2832041554456186, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1066, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 941, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 818, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8168402777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8827392120075047, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.848512173128945, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11726078799249531, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7100694444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7673545966228893, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7376014427412083, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8168402777777778, "step_test_time": 20.810358153999914, "step_plot_time": 13.015522008000062, "step_validation_cycle_time": 43.890257824999935} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..05ebf5fe3148d0d0baaced841c3403a7d10ff221 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 26.666666666666668, "train": {"Classification_binary": 0.2700510025024414, "Classification": 0.00041845691157504916, "Regression_pt": 0.23038171231746674, "Regression_eta": 0.04665890708565712, "Regression_sin_phi": 0.011044740676879883, "Regression_cos_phi": 0.011220389977097511, "Regression_energy": 0.26192161440849304, "Total": 2.6374242305755615, "Total_unweighted": 0.8316968238796107}, "train_group": {}, "valid": {"Classification_binary": 0.24281799793243408, "Classification": 0.0003601036344965299, "Regression_pt": 0.274752676486969, "Regression_eta": 0.04441367586453756, "Regression_sin_phi": 0.010711728284756342, "Regression_cos_phi": 0.011378618578116098, "Regression_energy": 0.28582807381947833, "Total": 2.5944862365722656, "Total_unweighted": 0.8702628746007879, "metrics/particle/class_1/efficiency": 0.9026817083358765, "metrics/particle/class_1/pid_accuracy": 0.988452672958374, "metrics/particle/class_2/efficiency": 0.06245461106300354, "metrics/particle/class_2/pid_accuracy": 0.029069768264889717, "metrics/particle/class_3/efficiency": 0.4383702278137207, "metrics/particle/class_3/pid_accuracy": 0.9704937934875488, "metrics/particle/class_4/efficiency": 0.8673469424247742, "metrics/particle/class_4/pid_accuracy": 0.250980406999588, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.2631579041481018, "metrics/particle/count/bias_mean": -19.30859375, "metrics/particle/count/mae": 19.35286521911621, "metrics/particle/count/prediction_mean": 60.2578125, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3014020025730133, "metrics/particle/event/energy_response_mean": 0.7013656497001648, "metrics/particle/event/met_abs_error": 14.281521797180176, "metrics/particle/event/scalar_pt_relative_abs_error": 0.24891264736652374, "metrics/particle/event/scalar_pt_response_mean": 0.7551148533821106, "metrics/particle/event/vector_pt_closure": 0.1134500801563263, "metrics/particle/matched/delta_eta_abs_mean": 0.007752655539661646, "metrics/particle/matched/delta_phi_abs_mean": 0.004260189365595579, "metrics/particle/matched/delta_r_mean": 0.009863398969173431, "metrics/particle/matched/energy_relative_abs_mean": 0.07911401242017746, "metrics/particle/matched/pid_accuracy": 0.9726391434669495, "metrics/particle/matched/pt_relative_abs_mean": 0.06998252868652344, "metrics/particle/matching/duplicate_fraction": 0.026837807148694992, "metrics/particle/matching/efficiency": 0.6650956273078918, "metrics/particle/matching/f1": 0.7569400072097778, "metrics/particle/matching/purity": 0.8782142996788025, "metrics/particle/matching/target_coverage_dr0p05": 0.6742271780967712, "metrics/particle/matching/target_coverage_dr0p10": 0.6924247741699219, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5862339735031128, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6804948449134827}, "step_train_time": 1.5372734069824219, "step_valid_time": 9.71088332599902, "step_total_time": 11065.186435699463, "elapsed_training_loop_seconds": 11065.186435699463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8129236632252216, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27789394874552154, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 602, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 471, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7521613832853026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.867109634551495, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8055555555555556, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.132890365448505, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6786743515850144, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.782392026578073, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7268518518518519, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7521613832853026, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8653173828060444, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.251400030508371, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1463, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1337, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1250, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8693107932379714, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9138755980861244, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8910363212262579, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0861244019138756, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.812743823146944, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8544087491455913, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8330556481172943, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8693107932379714, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8232796420068637, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2875143117152814, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1081, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 963, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 873, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8359375, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8908418131359852, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8625167935512763, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1091581868640148, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7578125, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8075855689176689, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7819077474249888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8359375, "step_test_time": 21.27908568500061, "step_plot_time": 13.044192801000463, "step_validation_cycle_time": 44.46451462799996} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_41000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_41000.json new file mode 100644 index 0000000000000000000000000000000000000000..a1de3a24b59a2f1f75bc5042242cc664a48686e2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_41000.json @@ -0,0 +1 @@ +{"epoch": 27.333333333333332, "train": {"Classification_binary": 0.23865675926208496, "Classification": 0.0005042693228460848, "Regression_pt": 0.24581581354141235, "Regression_eta": 0.02632366493344307, "Regression_sin_phi": 0.010481513105332851, "Regression_cos_phi": 0.011375556699931622, "Regression_energy": 0.2603117525577545, "Total": 2.346945285797119, "Total_unweighted": 0.7934693294228055}, "train_group": {}, "valid": {"Classification_binary": 0.2426062822341919, "Classification": 0.00036631776796032983, "Regression_pt": 0.27050284544626874, "Regression_eta": 0.04479833443959554, "Regression_sin_phi": 0.010783597826957703, "Regression_cos_phi": 0.011524484803279242, "Regression_energy": 0.2822159727414449, "Total": 2.609332879384359, "Total_unweighted": 0.8627978352596983, "metrics/particle/class_1/efficiency": 0.8935847878456116, "metrics/particle/class_1/pid_accuracy": 0.9890420436859131, "metrics/particle/class_2/efficiency": 0.05991285294294357, "metrics/particle/class_2/pid_accuracy": 0.024242423474788666, "metrics/particle/class_3/efficiency": 0.42475080490112305, "metrics/particle/class_3/pid_accuracy": 0.9716972708702087, "metrics/particle/class_4/efficiency": 0.8571428656578064, "metrics/particle/class_4/pid_accuracy": 0.2341269850730896, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.37719297409057617, "metrics/particle/count/bias_mean": -21.46224021911621, "metrics/particle/count/mae": 21.47786521911621, "metrics/particle/count/prediction_mean": 58.10416793823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3149917423725128, "metrics/particle/event/energy_response_mean": 0.6877451539039612, "metrics/particle/event/met_abs_error": 13.977668762207031, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26689547300338745, "metrics/particle/event/scalar_pt_response_mean": 0.7357461452484131, "metrics/particle/event/vector_pt_closure": 0.11205735057592392, "metrics/particle/matched/delta_eta_abs_mean": 0.007638879120349884, "metrics/particle/matched/delta_phi_abs_mean": 0.004281983245164156, "metrics/particle/matched/delta_r_mean": 0.009768490679562092, "metrics/particle/matched/energy_relative_abs_mean": 0.07928808778524399, "metrics/particle/matched/pid_accuracy": 0.9737157821655273, "metrics/particle/matched/pt_relative_abs_mean": 0.06684815138578415, "metrics/particle/matching/duplicate_fraction": 0.021221764385700226, "metrics/particle/matching/efficiency": 0.6543604135513306, "metrics/particle/matching/f1": 0.756372332572937, "metrics/particle/matching/purity": 0.8960648775100708, "metrics/particle/matching/target_coverage_dr0p05": 0.6604971885681152, "metrics/particle/matching/target_coverage_dr0p10": 0.6772055625915527, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5810136198997498, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.66741943359375}, "step_train_time": 1.5162038803100586, "step_valid_time": 9.774217996000516, "step_total_time": 11342.837671041489, "elapsed_training_loop_seconds": 11342.837671041489, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7981900249871322, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2752705962726767, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 592, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 465, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7463976945244957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.875, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8055987558320373, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.125, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.670028818443804, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.785472972972973, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7231726283048211, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7463976945244957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8533213435096957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25549905884698876, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1459, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1327, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1227, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8628088426527959, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9095270733379027, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8855522188855522, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09047292666209733, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7977893368010404, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8409869773817683, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8188188188188188, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8628088426527959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8154708146133551, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28267619118109266, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1073, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 958, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 863, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8315972222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8928238583410997, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.861123595505618, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10717614165890028, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7491319444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.804287045666356, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7757303370786517, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8315972222222222, "step_test_time": 21.58792709500085, "step_plot_time": 13.092436102000647, "step_validation_cycle_time": 44.8838905169996} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_42000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_42000.json new file mode 100644 index 0000000000000000000000000000000000000000..e9e5ca5648520681b9b63b0d84c506de049026bc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_42000.json @@ -0,0 +1 @@ +{"epoch": 28.0, "train": {"Classification_binary": 0.2218700647354126, "Classification": 0.00033825726131908596, "Regression_pt": 0.31022024154663086, "Regression_eta": 0.04338863119482994, "Regression_sin_phi": 0.006036950275301933, "Regression_cos_phi": 0.01120068971067667, "Regression_energy": 0.3066498041152954, "Total": 2.2835071086883545, "Total_unweighted": 0.8997046388394665}, "train_group": {}, "valid": {"Classification_binary": 0.24287782112757364, "Classification": 0.00036560014511148137, "Regression_pt": 0.2708131472269694, "Regression_eta": 0.04537675281365713, "Regression_sin_phi": 0.010745360205570856, "Regression_cos_phi": 0.011203328768412272, "Regression_energy": 0.2827470103899638, "Total": 2.59615691502889, "Total_unweighted": 0.8641290206772585, "metrics/particle/class_1/efficiency": 0.9017972946166992, "metrics/particle/class_1/pid_accuracy": 0.9888266324996948, "metrics/particle/class_2/efficiency": 0.06354393810033798, "metrics/particle/class_2/pid_accuracy": 0.051428571343421936, "metrics/particle/class_3/efficiency": 0.41664764285087585, "metrics/particle/class_3/pid_accuracy": 0.9651205539703369, "metrics/particle/class_4/efficiency": 0.8775510191917419, "metrics/particle/class_4/pid_accuracy": 0.22868217527866364, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.429824560880661, "metrics/particle/count/bias_mean": -20.38802146911621, "metrics/particle/count/mae": 20.40885353088379, "metrics/particle/count/prediction_mean": 59.17838668823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31388285756111145, "metrics/particle/event/energy_response_mean": 0.6904954314231873, "metrics/particle/event/met_abs_error": 14.343321800231934, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2612447142601013, "metrics/particle/event/scalar_pt_response_mean": 0.7438704967498779, "metrics/particle/event/vector_pt_closure": 0.11383453756570816, "metrics/particle/matched/delta_eta_abs_mean": 0.007796235848218203, "metrics/particle/matched/delta_phi_abs_mean": 0.004480348899960518, "metrics/particle/matched/delta_r_mean": 0.010038638487458229, "metrics/particle/matched/energy_relative_abs_mean": 0.07843897491693497, "metrics/particle/matched/pid_accuracy": 0.9717596173286438, "metrics/particle/matched/pt_relative_abs_mean": 0.06929869949817657, "metrics/particle/matching/duplicate_fraction": 0.023630883544683456, "metrics/particle/matching/efficiency": 0.6553913354873657, "metrics/particle/matching/f1": 0.7516986131668091, "metrics/particle/matching/purity": 0.8811855316162109, "metrics/particle/matching/target_coverage_dr0p05": 0.6626082062721252, "metrics/particle/matching/target_coverage_dr0p10": 0.6810021996498108, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5787716507911682, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6700705289840698}, "step_train_time": 1.445552110671997, "step_valid_time": 9.688136786999166, "step_total_time": 11620.78716468811, "elapsed_training_loop_seconds": 11620.78716468811, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8014650130453718, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2711726715986006, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 592, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 462, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7521613832853026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8817567567567568, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8118195956454122, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11824324324324324, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6657060518731989, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7804054054054054, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.71850699844479, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7521613832853026, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8493446004348748, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24732940576511353, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1461, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1331, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1240, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8654096228868661, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9110198494182067, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8876292097365789, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08898015058179329, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8062418725617685, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8487337440109514, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8269423141047015, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8654096228868661, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8117326125410153, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2830680382846257, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1079, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 957, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 867, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8307291666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8869323447636701, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8579112505602868, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11306765523632993, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7526041666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8035217794253939, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7772299417301658, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8307291666666666, "step_test_time": 21.672366054999657, "step_plot_time": 13.230460996999682, "step_validation_cycle_time": 45.0340556329993} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_43000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_43000.json new file mode 100644 index 0000000000000000000000000000000000000000..78f66e191b471939d9715eaea8cb3281dca40430 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_43000.json @@ -0,0 +1 @@ +{"epoch": 28.666666666666668, "train": {"Classification_binary": 0.2363649159669876, "Classification": 0.0004133973561692983, "Regression_pt": 0.33909136056900024, "Regression_eta": 0.03248884156346321, "Regression_sin_phi": 0.008354166522622108, "Regression_cos_phi": 0.0123361274600029, "Regression_energy": 0.32801151275634766, "Total": 2.4191689491271973, "Total_unweighted": 0.957060322194593}, "train_group": {}, "valid": {"Classification_binary": 0.24273584286371866, "Classification": 0.00036726232307652634, "Regression_pt": 0.27235958973566693, "Regression_eta": 0.04856036106745402, "Regression_sin_phi": 0.011009007692337036, "Regression_cos_phi": 0.011446728060642878, "Regression_energy": 0.28541980187098187, "Total": 2.671227773030599, "Total_unweighted": 0.8718985936138779, "metrics/particle/class_1/efficiency": 0.898733377456665, "metrics/particle/class_1/pid_accuracy": 0.9879099130630493, "metrics/particle/class_2/efficiency": 0.06535948067903519, "metrics/particle/class_2/pid_accuracy": 0.06111111119389534, "metrics/particle/class_3/efficiency": 0.41653352975845337, "metrics/particle/class_3/pid_accuracy": 0.9654762744903564, "metrics/particle/class_4/efficiency": 0.8741496801376343, "metrics/particle/class_4/pid_accuracy": 0.3035019338130951, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.31578946113586426, "metrics/particle/count/bias_mean": -21.01822853088379, "metrics/particle/count/mae": 21.046875, "metrics/particle/count/prediction_mean": 58.54817581176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3062438666820526, "metrics/particle/event/energy_response_mean": 0.6968863606452942, "metrics/particle/event/met_abs_error": 14.123518943786621, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25433582067489624, "metrics/particle/event/scalar_pt_response_mean": 0.7499551773071289, "metrics/particle/event/vector_pt_closure": 0.11235811561346054, "metrics/particle/matched/delta_eta_abs_mean": 0.007752958219498396, "metrics/particle/matched/delta_phi_abs_mean": 0.004345479421317577, "metrics/particle/matched/delta_r_mean": 0.009903579950332642, "metrics/particle/matched/energy_relative_abs_mean": 0.08054798096418381, "metrics/particle/matched/pid_accuracy": 0.9712662100791931, "metrics/particle/matched/pt_relative_abs_mean": 0.06918342411518097, "metrics/particle/matching/duplicate_fraction": 0.02426331676542759, "metrics/particle/matching/efficiency": 0.6538203358650208, "metrics/particle/matching/f1": 0.7533184885978699, "metrics/particle/matching/purity": 0.8885354995727539, "metrics/particle/matching/target_coverage_dr0p05": 0.660628080368042, "metrics/particle/matching/target_coverage_dr0p10": 0.6779419779777527, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5789843797683716, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.668401300907135}, "step_train_time": 1.4828848838806152, "step_valid_time": 9.727481174999411, "step_total_time": 11900.124316692352, "elapsed_training_loop_seconds": 11900.124316692352, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8055286111051913, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2738708635041791, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 597, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 517, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 462, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7449567723342939, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8659966499162479, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8009295120061968, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13400335008375208, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6657060518731989, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7738693467336684, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.715724244771495, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7449567723342939, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8643091329619321, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2536487919762309, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1455, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1329, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1242, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.864109232769831, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9134020618556701, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8880721683929168, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0865979381443299, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8075422626788037, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8536082474226804, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8299365185432677, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.864109232769831, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8170426890986947, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2847449845003752, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 958, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 869, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8315972222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8886827458256029, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8591928251121076, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11131725417439703, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7543402777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8061224489795918, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7793721973094171, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8315972222222222, "step_test_time": 21.419234822998988, "step_plot_time": 13.164768975999323, "step_validation_cycle_time": 44.76322131699999} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_44000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_44000.json new file mode 100644 index 0000000000000000000000000000000000000000..059e37faa572464466b4f3e5481db7336c2950eb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_44000.json @@ -0,0 +1 @@ +{"epoch": 29.333333333333332, "train": {"Classification_binary": 0.22724393010139465, "Classification": 0.00033619068562984467, "Regression_pt": 0.3051629066467285, "Regression_eta": 0.08242134749889374, "Regression_sin_phi": 0.005158016923815012, "Regression_cos_phi": 0.007446257397532463, "Regression_energy": 0.34595760703086853, "Total": 2.540051221847534, "Total_unweighted": 0.9737262562848628}, "train_group": {}, "valid": {"Classification_binary": 0.24150355656941733, "Classification": 0.0003595856639246146, "Regression_pt": 0.27112215757369995, "Regression_eta": 0.04672574996948242, "Regression_sin_phi": 0.011041225244601568, "Regression_cos_phi": 0.011052745083967844, "Regression_energy": 0.28272678454717, "Total": 2.620621840159098, "Total_unweighted": 0.8645318046522636, "metrics/particle/class_1/efficiency": 0.907767117023468, "metrics/particle/class_1/pid_accuracy": 0.9893176555633545, "metrics/particle/class_2/efficiency": 0.06281772255897522, "metrics/particle/class_2/pid_accuracy": 0.023121386766433716, "metrics/particle/class_3/efficiency": 0.42208778858184814, "metrics/particle/class_3/pid_accuracy": 0.9688147902488708, "metrics/particle/class_4/efficiency": 0.8877550959587097, "metrics/particle/class_4/pid_accuracy": 0.29118773341178894, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.3245614171028137, "metrics/particle/count/bias_mean": -20.20963478088379, "metrics/particle/count/mae": 20.24869728088379, "metrics/particle/count/prediction_mean": 59.35676956176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31726935505867004, "metrics/particle/event/energy_response_mean": 0.683874785900116, "metrics/particle/event/met_abs_error": 14.0862455368042, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2660696506500244, "metrics/particle/event/scalar_pt_response_mean": 0.7357246279716492, "metrics/particle/event/vector_pt_closure": 0.11222612112760544, "metrics/particle/matched/delta_eta_abs_mean": 0.008417082950472832, "metrics/particle/matched/delta_phi_abs_mean": 0.004244007170200348, "metrics/particle/matched/delta_r_mean": 0.010428224690258503, "metrics/particle/matched/energy_relative_abs_mean": 0.07649756968021393, "metrics/particle/matched/pid_accuracy": 0.9731563329696655, "metrics/particle/matched/pt_relative_abs_mean": 0.06625167280435562, "metrics/particle/matching/duplicate_fraction": 0.022989515215158463, "metrics/particle/matching/efficiency": 0.6608408093452454, "metrics/particle/matching/f1": 0.7569756507873535, "metrics/particle/matching/purity": 0.8858421444892883, "metrics/particle/matching/target_coverage_dr0p05": 0.6698086857795715, "metrics/particle/matching/target_coverage_dr0p10": 0.6864843368530273, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5874122381210327, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6753890514373779}, "step_train_time": 1.4425685405731201, "step_valid_time": 9.674302320001516, "step_total_time": 12177.151175498962, "elapsed_training_loop_seconds": 12177.151175498962, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7960741726179112, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2700523672569328, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 587, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 516, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 464, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7435158501440923, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.879045996592845, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8056206088992974, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12095400340715502, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6685878962536023, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7904599659284497, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7244340359094458, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7435158501440923, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8533816027674891, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25255736562816167, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1469, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1334, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1243, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8673602080624188, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9081007488087134, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8872630528766212, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09189925119128659, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8081924577373212, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8461538461538461, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.826737612238111, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8673602080624188, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8154411821578315, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27964114908351423, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 865, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8324652777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8896103896103896, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8600896860986547, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11038961038961038, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7508680555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8024118738404453, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7757847533632287, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8324652777777778, "step_test_time": 21.73297994099994, "step_plot_time": 13.134673633001512, "step_validation_cycle_time": 44.99365846499859} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_45000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_45000.json new file mode 100644 index 0000000000000000000000000000000000000000..ba87e1a1c9dc87a927fea3c6caf31e60aae03e70 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_45000.json @@ -0,0 +1 @@ +{"epoch": 30.0, "train": {"Classification_binary": 0.2323378622531891, "Classification": 0.00044336539576761425, "Regression_pt": 0.22276853024959564, "Regression_eta": 0.02599840797483921, "Regression_sin_phi": 0.009607750922441483, "Regression_cos_phi": 0.00845585111528635, "Regression_energy": 0.23556002974510193, "Total": 2.0649020671844482, "Total_unweighted": 0.7351717976562213}, "train_group": {}, "valid": {"Classification_binary": 0.24265970786412558, "Classification": 0.00036390832004447776, "Regression_pt": 0.26962852478027344, "Regression_eta": 0.04408895969390869, "Regression_sin_phi": 0.010832099864880243, "Regression_cos_phi": 0.01169844220081965, "Regression_energy": 0.280950923760732, "Total": 2.611512025197347, "Total_unweighted": 0.860222566484784, "metrics/particle/class_1/efficiency": 0.9062825441360474, "metrics/particle/class_1/pid_accuracy": 0.9848738312721252, "metrics/particle/class_2/efficiency": 0.06971677392721176, "metrics/particle/class_2/pid_accuracy": 0.0520833320915699, "metrics/particle/class_3/efficiency": 0.4704025089740753, "metrics/particle/class_3/pid_accuracy": 0.9733926653862, "metrics/particle/class_4/efficiency": 0.8809523582458496, "metrics/particle/class_4/pid_accuracy": 0.31660231947898865, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.5087719559669495, "metrics/particle/count/bias_mean": -17.32161521911621, "metrics/particle/count/mae": 17.36849021911621, "metrics/particle/count/prediction_mean": 62.24479293823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.2946666479110718, "metrics/particle/event/energy_response_mean": 0.7082400918006897, "metrics/particle/event/met_abs_error": 14.375911712646484, "metrics/particle/event/scalar_pt_relative_abs_error": 0.23990289866924286, "metrics/particle/event/scalar_pt_response_mean": 0.7638804316520691, "metrics/particle/event/vector_pt_closure": 0.11206678301095963, "metrics/particle/matched/delta_eta_abs_mean": 0.008302809670567513, "metrics/particle/matched/delta_phi_abs_mean": 0.004316276870667934, "metrics/particle/matched/delta_r_mean": 0.010357344523072243, "metrics/particle/matched/energy_relative_abs_mean": 0.0800347700715065, "metrics/particle/matched/pid_accuracy": 0.9716976881027222, "metrics/particle/matched/pt_relative_abs_mean": 0.07109448313713074, "metrics/particle/matching/duplicate_fraction": 0.029411764815449715, "metrics/particle/matching/efficiency": 0.6811330914497375, "metrics/particle/matching/f1": 0.7643305063247681, "metrics/particle/matching/purity": 0.8706802725791931, "metrics/particle/matching/target_coverage_dr0p05": 0.6906737089157104, "metrics/particle/matching/target_coverage_dr0p10": 0.7090349793434143, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5998494625091553, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6980215311050415}, "step_train_time": 1.477649450302124, "step_valid_time": 9.741157673999624, "step_total_time": 12456.309059858322, "elapsed_training_loop_seconds": 12456.309059858322, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8248533791704139, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26901068722884836, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 598, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 521, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 476, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7507204610951008, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8712374581939799, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8065015479876161, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12876254180602006, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6858789625360231, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7959866220735786, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7368421052631579, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7507204610951008, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8947682703844675, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2346651739599479, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1470, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1342, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1262, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8725617685305592, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9129251700680272, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8922872340425532, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08707482993197278, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8205461638491548, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8585034013605443, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.839095744680851, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8725617685305592, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8381446331061991, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2990007747287613, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1088, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 973, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 889, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8446180555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8943014705882353, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.86875, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1056985294117647, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7717013888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8170955882352942, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.79375, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8446180555555556, "step_test_time": 21.97673100799875, "step_plot_time": 13.050873652999144, "step_validation_cycle_time": 45.22581189700031} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_46000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_46000.json new file mode 100644 index 0000000000000000000000000000000000000000..e25002300a0621fd7895a0221274f26773be8e78 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_46000.json @@ -0,0 +1 @@ +{"epoch": 30.666666666666668, "train": {"Classification_binary": 0.24603919684886932, "Classification": 0.00037471757968887687, "Regression_pt": 0.33257803320884705, "Regression_eta": 0.03531450405716896, "Regression_sin_phi": 0.004417306277900934, "Regression_cos_phi": 0.005976497195661068, "Regression_energy": 0.3387467861175537, "Total": 1.8364160060882568, "Total_unweighted": 0.9634470412856899}, "train_group": {}, "valid": {"Classification_binary": 0.24159262577692667, "Classification": 0.00036067360391219455, "Regression_pt": 0.26753805081049603, "Regression_eta": 0.044369906187057495, "Regression_sin_phi": 0.011309302101532618, "Regression_cos_phi": 0.011320073157548904, "Regression_energy": 0.2793735861778259, "Total": 2.6188162167867026, "Total_unweighted": 0.8558642178152999, "metrics/particle/class_1/efficiency": 0.9043557643890381, "metrics/particle/class_1/pid_accuracy": 0.9877755045890808, "metrics/particle/class_2/efficiency": 0.06608569622039795, "metrics/particle/class_2/pid_accuracy": 0.032967034727334976, "metrics/particle/class_3/efficiency": 0.4468157887458801, "metrics/particle/class_3/pid_accuracy": 0.9711366295814514, "metrics/particle/class_4/efficiency": 0.8741496801376343, "metrics/particle/class_4/pid_accuracy": 0.2684824764728546, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.35087719559669495, "metrics/particle/count/bias_mean": -19.0625, "metrics/particle/count/mae": 19.07291603088379, "metrics/particle/count/prediction_mean": 60.50390625, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3018006384372711, "metrics/particle/event/energy_response_mean": 0.701572835445404, "metrics/particle/event/met_abs_error": 14.668482780456543, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2493944764137268, "metrics/particle/event/scalar_pt_response_mean": 0.7557260394096375, "metrics/particle/event/vector_pt_closure": 0.11405164003372192, "metrics/particle/matched/delta_eta_abs_mean": 0.008452421054244041, "metrics/particle/matched/delta_phi_abs_mean": 0.004253312945365906, "metrics/particle/matched/delta_r_mean": 0.010453861206769943, "metrics/particle/matched/energy_relative_abs_mean": 0.07579879462718964, "metrics/particle/matched/pid_accuracy": 0.9724645018577576, "metrics/particle/matched/pt_relative_abs_mean": 0.06683389097452164, "metrics/particle/matching/duplicate_fraction": 0.026728646829724312, "metrics/particle/matching/efficiency": 0.6697923541069031, "metrics/particle/matching/f1": 0.7609459757804871, "metrics/particle/matching/purity": 0.8808186650276184, "metrics/particle/matching/target_coverage_dr0p05": 0.6756345629692078, "metrics/particle/matching/target_coverage_dr0p10": 0.6949121952056885, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5960528254508972, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6857643127441406}, "step_train_time": 1.5102427005767822, "step_valid_time": 9.685146477000671, "step_total_time": 12734.738800048828, "elapsed_training_loop_seconds": 12734.738800048828, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8078777421658772, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2616376595291937, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 597, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 524, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 474, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7550432276657061, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8777219430485762, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8117738187451587, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12227805695142378, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6829971181556196, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7939698492462312, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7343144848954299, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7550432276657061, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8766962648710469, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24781389446250746, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1463, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1338, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1247, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8699609882964889, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9145591250854409, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8917027657447517, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08544087491455912, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8107932379713915, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8523581681476419, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8310563145618127, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8699609882964889, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8291763211854942, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27914203403031335, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1074, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 963, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 879, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8359375, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8966480446927374, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8652291105121294, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10335195530726257, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7630208333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8184357541899442, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7897574123989218, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8359375, "step_test_time": 21.35002105300009, "step_plot_time": 13.134633812000175, "step_validation_cycle_time": 44.610620890000064} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_47000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_47000.json new file mode 100644 index 0000000000000000000000000000000000000000..18eaad4b4489102decd1854fc9868f77499adf71 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_47000.json @@ -0,0 +1 @@ +{"epoch": 31.333333333333332, "train": {"Classification_binary": 0.2240932136774063, "Classification": 0.00034346626489423215, "Regression_pt": 0.2689562439918518, "Regression_eta": 0.029045717790722847, "Regression_sin_phi": 0.004071722738444805, "Regression_cos_phi": 0.0048024654388427734, "Regression_energy": 0.28104692697525024, "Total": 1.5613170862197876, "Total_unweighted": 0.812359756877413}, "train_group": {}, "valid": {"Classification_binary": 0.240879495938619, "Classification": 0.00036060930384943884, "Regression_pt": 0.26352715492248535, "Regression_eta": 0.04572318991025289, "Regression_sin_phi": 0.010682982703049978, "Regression_cos_phi": 0.011230124781529108, "Regression_energy": 0.27694549163182575, "Total": 2.587134043375651, "Total_unweighted": 0.8493490491916116, "metrics/particle/class_1/efficiency": 0.8993967175483704, "metrics/particle/class_1/pid_accuracy": 0.9894640445709229, "metrics/particle/class_2/efficiency": 0.060639072209596634, "metrics/particle/class_2/pid_accuracy": 0.0359281450510025, "metrics/particle/class_3/efficiency": 0.4288594722747803, "metrics/particle/class_3/pid_accuracy": 0.971347451210022, "metrics/particle/class_4/efficiency": 0.8809523582458496, "metrics/particle/class_4/pid_accuracy": 0.25482624769210815, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.28947368264198303, "metrics/particle/count/bias_mean": -20.79166603088379, "metrics/particle/count/mae": 20.80208396911621, "metrics/particle/count/prediction_mean": 58.77473831176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31412994861602783, "metrics/particle/event/energy_response_mean": 0.6894052624702454, "metrics/particle/event/met_abs_error": 14.377368927001953, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2629166543483734, "metrics/particle/event/scalar_pt_response_mean": 0.7412516474723816, "metrics/particle/event/vector_pt_closure": 0.11318068951368332, "metrics/particle/matched/delta_eta_abs_mean": 0.008169949986040592, "metrics/particle/matched/delta_phi_abs_mean": 0.004233573097735643, "metrics/particle/matched/delta_r_mean": 0.01017911359667778, "metrics/particle/matched/energy_relative_abs_mean": 0.07874175906181335, "metrics/particle/matched/pid_accuracy": 0.973738431930542, "metrics/particle/matched/pt_relative_abs_mean": 0.06766737997531891, "metrics/particle/matching/duplicate_fraction": 0.023261481896042824, "metrics/particle/matching/efficiency": 0.6592861413955688, "metrics/particle/matching/f1": 0.7583720684051514, "metrics/particle/matching/purity": 0.8925098180770874, "metrics/particle/matching/target_coverage_dr0p05": 0.6649320125579834, "metrics/particle/matching/target_coverage_dr0p10": 0.6820822358131409, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5856448411941528, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6726070642471313}, "step_train_time": 1.452845573425293, "step_valid_time": 9.67936005500087, "step_total_time": 13013.169122219086, "elapsed_training_loop_seconds": 13013.169122219086, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7906914180996407, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26585916874485893, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 594, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 513, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 459, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.739193083573487, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8636363636363636, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.796583850931677, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13636363636363635, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6613832853025937, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7727272727272727, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7127329192546584, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.739193083573487, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8595679640054745, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2472890362365764, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1463, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1329, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1235, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.864109232769831, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9084073820915927, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8857047650783072, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09159261790840738, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8029908972691807, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8441558441558441, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8230589803398867, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.864109232769831, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8125620295418506, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2762070841917259, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 961, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 868, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8342013888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.891465677179963, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8618834080717489, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10853432282003711, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7534722222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8051948051948052, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7784753363228699, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8342013888888888, "step_test_time": 21.72508391500014, "step_plot_time": 13.04707391500051, "step_validation_cycle_time": 44.8955096360005} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_48000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_48000.json new file mode 100644 index 0000000000000000000000000000000000000000..b9b621c237898bd25be7f47b83a5dd7c617c8416 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_48000.json @@ -0,0 +1 @@ +{"epoch": 32.0, "train": {"Classification_binary": 0.2305738925933838, "Classification": 0.0003997634048573673, "Regression_pt": 0.2871992290019989, "Regression_eta": 0.020310891792178154, "Regression_sin_phi": 0.00760574359446764, "Regression_cos_phi": 0.006922189611941576, "Regression_energy": 0.2920207977294922, "Total": 1.833797574043274, "Total_unweighted": 0.8450325077283196}, "train_group": {}, "valid": {"Classification_binary": 0.24129303296407065, "Classification": 0.000358957564458251, "Regression_pt": 0.2641369303067525, "Regression_eta": 0.04336811602115631, "Regression_sin_phi": 0.010183297097682953, "Regression_cos_phi": 0.011251591145992279, "Regression_energy": 0.27642927567164105, "Total": 2.5263942082722983, "Total_unweighted": 0.8470212007717539, "metrics/particle/class_1/efficiency": 0.9041031002998352, "metrics/particle/class_1/pid_accuracy": 0.9889249801635742, "metrics/particle/class_2/efficiency": 0.061365287750959396, "metrics/particle/class_2/pid_accuracy": 0.023668639361858368, "metrics/particle/class_3/efficiency": 0.4190063178539276, "metrics/particle/class_3/pid_accuracy": 0.9672235250473022, "metrics/particle/class_4/efficiency": 0.8979591727256775, "metrics/particle/class_4/pid_accuracy": 0.32575756311416626, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.42105263471603394, "metrics/particle/count/bias_mean": -20.41927146911621, "metrics/particle/count/mae": 20.4375, "metrics/particle/count/prediction_mean": 59.14713668823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3183951675891876, "metrics/particle/event/energy_response_mean": 0.6871576905250549, "metrics/particle/event/met_abs_error": 14.600166320800781, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26680049300193787, "metrics/particle/event/scalar_pt_response_mean": 0.7381216883659363, "metrics/particle/event/vector_pt_closure": 0.11246693134307861, "metrics/particle/matched/delta_eta_abs_mean": 0.007721754722297192, "metrics/particle/matched/delta_phi_abs_mean": 0.004443602170795202, "metrics/particle/matched/delta_r_mean": 0.0099183926358819, "metrics/particle/matched/energy_relative_abs_mean": 0.07542590796947479, "metrics/particle/matched/pid_accuracy": 0.9729494452476501, "metrics/particle/matched/pt_relative_abs_mean": 0.06581009179353714, "metrics/particle/matching/duplicate_fraction": 0.022344524040818214, "metrics/particle/matching/efficiency": 0.6576005816459656, "metrics/particle/matching/f1": 0.7544024586677551, "metrics/particle/matching/purity": 0.8846229910850525, "metrics/particle/matching/target_coverage_dr0p05": 0.6645883321762085, "metrics/particle/matching/target_coverage_dr0p10": 0.6821313500404358, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5837628841400146, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6714451909065247}, "step_train_time": 1.4940359592437744, "step_valid_time": 9.734607869999309, "step_total_time": 13291.003299713135, "elapsed_training_loop_seconds": 13291.003299713135, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7992348495826083, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26562137976298106, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 597, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 462, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7463976945244957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8676716917922948, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8024786986831913, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1323283082077052, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6657060518731989, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7738693467336684, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.715724244771495, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7463976945244957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8473965928569147, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24640256014676598, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1452, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1316, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1222, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8556566970091027, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.90633608815427, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8802675585284281, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09366391184573003, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7945383615084526, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8415977961432507, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8173913043478261, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8556566970091027, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8075804420005438, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28478188477997135, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1073, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 957, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 847, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8307291666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8918918918918919, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8602247191011236, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10810810810810811, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7352430555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7893755824790307, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7613483146067416, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8307291666666666, "step_test_time": 21.678664192000724, "step_plot_time": 13.187466003999361, "step_validation_cycle_time": 45.110505668999394} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_49000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_49000.json new file mode 100644 index 0000000000000000000000000000000000000000..97d33dd1a53c475c34f47b0dbbaeef03ae581e9c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_49000.json @@ -0,0 +1 @@ +{"epoch": 32.666666666666664, "train": {"Classification_binary": 0.22625821828842163, "Classification": 0.0003853758971672505, "Regression_pt": 0.2476446032524109, "Regression_eta": 0.07809785008430481, "Regression_sin_phi": 0.006434795446693897, "Regression_cos_phi": 0.008233330212533474, "Regression_energy": 0.29022255539894104, "Total": 2.55495023727417, "Total_unweighted": 0.857276728580473}, "train_group": {}, "valid": {"Classification_binary": 0.2414273222287496, "Classification": 0.00035963925377776224, "Regression_pt": 0.26399463415145874, "Regression_eta": 0.04680077234903971, "Regression_sin_phi": 0.010868151982625326, "Regression_cos_phi": 0.01128352185090383, "Regression_energy": 0.2775278886159261, "Total": 2.6168320973714194, "Total_unweighted": 0.8522619304324812, "metrics/particle/class_1/efficiency": 0.9046400785446167, "metrics/particle/class_1/pid_accuracy": 0.987080991268158, "metrics/particle/class_2/efficiency": 0.06753812730312347, "metrics/particle/class_2/pid_accuracy": 0.04301075264811516, "metrics/particle/class_3/efficiency": 0.43224531412124634, "metrics/particle/class_3/pid_accuracy": 0.9678753614425659, "metrics/particle/class_4/efficiency": 0.8469387888908386, "metrics/particle/class_4/pid_accuracy": 0.39759036898612976, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.41228070855140686, "metrics/particle/count/bias_mean": -18.31510353088379, "metrics/particle/count/mae": 18.359375, "metrics/particle/count/prediction_mean": 61.25130081176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.2988622486591339, "metrics/particle/event/energy_response_mean": 0.707188606262207, "metrics/particle/event/met_abs_error": 14.278873443603516, "metrics/particle/event/scalar_pt_relative_abs_error": 0.24302338063716888, "metrics/particle/event/scalar_pt_response_mean": 0.7621657252311707, "metrics/particle/event/vector_pt_closure": 0.11501351743936539, "metrics/particle/matched/delta_eta_abs_mean": 0.008604694157838821, "metrics/particle/matched/delta_phi_abs_mean": 0.004122531972825527, "metrics/particle/matched/delta_r_mean": 0.010539301671087742, "metrics/particle/matched/energy_relative_abs_mean": 0.07963641732931137, "metrics/particle/matched/pid_accuracy": 0.9721338748931885, "metrics/particle/matched/pt_relative_abs_mean": 0.07087749242782593, "metrics/particle/matching/duplicate_fraction": 0.02982504665851593, "metrics/particle/matching/efficiency": 0.6636064648628235, "metrics/particle/matching/f1": 0.7499167919158936, "metrics/particle/matching/purity": 0.8620352745056152, "metrics/particle/matching/target_coverage_dr0p05": 0.6796111464500427, "metrics/particle/matching/target_coverage_dr0p10": 0.6989870071411133, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5813736319541931, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6805276274681091}, "step_train_time": 1.3567323684692383, "step_valid_time": 9.729231811001227, "step_total_time": 13569.672111034393, "elapsed_training_loop_seconds": 13569.672111034393, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8248304227867558, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.267182652196681, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 601, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 523, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 474, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7536023054755043, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.870216306156406, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8077220077220078, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.129783693843594, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6829971181556196, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7886855241264559, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7320463320463321, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7536023054755043, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8705637917406673, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24140635660497178, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1472, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1344, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1257, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8738621586475943, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9130434782608695, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8930232558139535, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08695652173913043, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.817295188556567, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8539402173913043, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8352159468438538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8738621586475943, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8377682228433833, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.29143077441165166, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1089, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 966, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 885, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8385416666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8870523415977961, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8621151271753681, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11294765840220386, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7682291666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8126721763085399, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7898259705488622, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8385416666666666, "step_test_time": 21.570506751999346, "step_plot_time": 13.042760786000144, "step_validation_cycle_time": 44.79173773900038} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_5000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..f86b61c952831d9976c9715a815bae0983599766 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 3.3333333333333335, "train": {"Classification_binary": 0.26108554005622864, "Classification": 0.0006422677543014288, "Regression_pt": 0.3603813648223877, "Regression_eta": 0.08668571710586548, "Regression_sin_phi": 0.018215211108326912, "Regression_cos_phi": 0.027671970427036285, "Regression_energy": 0.38519415259361267, "Total": 4.777807235717773, "Total_unweighted": 1.139876223867759}, "train_group": {}, "valid": {"Classification_binary": 0.2736000418663025, "Classification": 0.0005161028432970246, "Regression_pt": 0.3967876434326172, "Regression_eta": 0.05915791789690653, "Regression_sin_phi": 0.014176350086927414, "Regression_cos_phi": 0.014415739725033442, "Regression_energy": 0.41264541943868, "Total": 3.4040956497192383, "Total_unweighted": 1.1712992152897639, "metrics/particle/class_1/efficiency": 0.8747591376304626, "metrics/particle/class_1/pid_accuracy": 0.9853397607803345, "metrics/particle/class_2/efficiency": 0.056644879281520844, "metrics/particle/class_2/pid_accuracy": 0.006410256493836641, "metrics/particle/class_3/efficiency": 0.3278931677341461, "metrics/particle/class_3/pid_accuracy": 0.9530107975006104, "metrics/particle/class_4/efficiency": 0.8435373902320862, "metrics/particle/class_4/pid_accuracy": 0.02822580561041832, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.1315789520740509, "metrics/particle/count/bias_mean": -23.79036521911621, "metrics/particle/count/mae": 23.82942771911621, "metrics/particle/count/prediction_mean": 55.77604293823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3216573894023895, "metrics/particle/event/energy_response_mean": 0.6811307072639465, "metrics/particle/event/met_abs_error": 15.16123104095459, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2735125720500946, "metrics/particle/event/scalar_pt_response_mean": 0.7309221625328064, "metrics/particle/event/vector_pt_closure": 0.11634140461683273, "metrics/particle/matched/delta_eta_abs_mean": 0.009020178578794003, "metrics/particle/matched/delta_phi_abs_mean": 0.004032040946185589, "metrics/particle/matched/delta_r_mean": 0.01081488374620676, "metrics/particle/matched/energy_relative_abs_mean": 0.08330582082271576, "metrics/particle/matched/pid_accuracy": 0.9645407199859619, "metrics/particle/matched/pt_relative_abs_mean": 0.07375743240118027, "metrics/particle/matching/duplicate_fraction": 0.024301989004015923, "metrics/particle/matching/efficiency": 0.6027296185493469, "metrics/particle/matching/f1": 0.7086768746376038, "metrics/particle/matching/purity": 0.8598141670227051, "metrics/particle/matching/target_coverage_dr0p05": 0.6043006777763367, "metrics/particle/matching/target_coverage_dr0p10": 0.6283732056617737, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5414437055587769, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6234637498855591}, "step_train_time": 1.2294833660125732, "step_valid_time": 9.65813295099997, "step_total_time": 1353.7300968170166, "elapsed_training_loop_seconds": 1353.7300968170166, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8024987392368028, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.288925656280278, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 516, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 457, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7435158501440923, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.882051282051282, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8068803752931978, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11794871794871795, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6585014409221902, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7811965811965812, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7146207974980453, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7435158501440923, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8535057635724563, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26827850787091756, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1466, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1314, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1198, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8543563068920677, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8963165075034106, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8748335552596538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10368349249658936, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7789336801040312, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8171896316507503, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7976031957390146, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8543563068920677, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8125837514163806, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28408803487872114, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1066, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 953, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 839, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8272569444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8939962476547842, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8593327321911632, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10600375234521577, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7282986111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7870544090056285, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7565374211000901, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8272569444444444, "step_test_time": 20.589906220000103, "step_plot_time": 12.63533111300012, "step_validation_cycle_time": 43.324330101999976} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..489d026b26782c3cade72b79c1cb57e35ea79569 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 33.333333333333336, "train": {"Classification_binary": 0.269087016582489, "Classification": 0.00030276135657913983, "Regression_pt": 0.28235894441604614, "Regression_eta": 0.03609549254179001, "Regression_sin_phi": 0.011839732527732849, "Regression_cos_phi": 0.009387413039803505, "Regression_energy": 0.3113867938518524, "Total": 2.4710559844970703, "Total_unweighted": 0.9204581543162931}, "train_group": {}, "valid": {"Classification_binary": 0.24074647823969522, "Classification": 0.0003549353762840231, "Regression_pt": 0.2648872534434001, "Regression_eta": 0.043972874681154885, "Regression_sin_phi": 0.01031984326740106, "Regression_cos_phi": 0.011210567007462183, "Regression_energy": 0.27758387724558514, "Total": 2.5399382909139, "Total_unweighted": 0.8490758292609826, "metrics/particle/class_1/efficiency": 0.8947850465774536, "metrics/particle/class_1/pid_accuracy": 0.9893391728401184, "metrics/particle/class_2/efficiency": 0.061002179980278015, "metrics/particle/class_2/pid_accuracy": 0.02976190485060215, "metrics/particle/class_3/efficiency": 0.43106597661972046, "metrics/particle/class_3/pid_accuracy": 0.9691112637519836, "metrics/particle/class_4/efficiency": 0.8979591727256775, "metrics/particle/class_4/pid_accuracy": 0.2916666567325592, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.25438597798347473, "metrics/particle/count/bias_mean": -21.07161521911621, "metrics/particle/count/mae": 21.08203125, "metrics/particle/count/prediction_mean": 58.49479293823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.30756574869155884, "metrics/particle/event/energy_response_mean": 0.6950960755348206, "metrics/particle/event/met_abs_error": 14.322991371154785, "metrics/particle/event/scalar_pt_relative_abs_error": 0.255530446767807, "metrics/particle/event/scalar_pt_response_mean": 0.7482929825782776, "metrics/particle/event/vector_pt_closure": 0.11394301056861877, "metrics/particle/matched/delta_eta_abs_mean": 0.00758574903011322, "metrics/particle/matched/delta_phi_abs_mean": 0.004094463773071766, "metrics/particle/matched/delta_r_mean": 0.009594428353011608, "metrics/particle/matched/energy_relative_abs_mean": 0.08222682774066925, "metrics/particle/matched/pid_accuracy": 0.9729635715484619, "metrics/particle/matched/pt_relative_abs_mean": 0.0689319297671318, "metrics/particle/matching/duplicate_fraction": 0.02337280660867691, "metrics/particle/matching/efficiency": 0.6579442620277405, "metrics/particle/matching/f1": 0.7583631277084351, "metrics/particle/matching/purity": 0.8949559330940247, "metrics/particle/matching/target_coverage_dr0p05": 0.6634427905082703, "metrics/particle/matching/target_coverage_dr0p10": 0.6811822056770325, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5830264687538147, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6715433597564697}, "step_train_time": 1.430643081665039, "step_valid_time": 9.771022013999755, "step_total_time": 13847.670347690582, "elapsed_training_loop_seconds": 13847.670347690582, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8015933560479991, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2702947097373565, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 591, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 469, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7463976945244957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8764805414551607, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8062256809338522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12351945854483926, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6757925072046109, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7935702199661591, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7299610894941634, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7463976945244957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8638342965500172, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25710939536443755, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1463, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1336, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1244, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8686605981794538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9131920710868079, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8903698767077641, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08680792891319207, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8088426527958388, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8503075871496925, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8290569810063312, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8686605981794538, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8196983381347184, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2824947420299677, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 965, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 883, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8376736111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8951762523191095, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8654708520179372, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10482374768089053, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7664930555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8191094619666048, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7919282511210762, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8376736111111112, "step_test_time": 21.661976510000386, "step_plot_time": 13.050074703000064, "step_validation_cycle_time": 44.93641590699917} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_51000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_51000.json new file mode 100644 index 0000000000000000000000000000000000000000..1d5e86694c24a6ffd0f7d4f51e55f864b954632f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_51000.json @@ -0,0 +1 @@ +{"epoch": 34.0, "train": {"Classification_binary": 0.23883742094039917, "Classification": 0.0003616814501583576, "Regression_pt": 0.2619122564792633, "Regression_eta": 0.030454987660050392, "Regression_sin_phi": 0.011471928097307682, "Regression_cos_phi": 0.005582442041486502, "Regression_energy": 0.2687283754348755, "Total": 2.098992347717285, "Total_unweighted": 0.8173490921035409}, "train_group": {}, "valid": {"Classification_binary": 0.24061171213785806, "Classification": 0.0003607814045002063, "Regression_pt": 0.2676785389582316, "Regression_eta": 0.045215229193369545, "Regression_sin_phi": 0.01073105384906133, "Regression_cos_phi": 0.011997915804386139, "Regression_energy": 0.2798731525739034, "Total": 2.6326777140299478, "Total_unweighted": 0.8564683839213103, "metrics/particle/class_1/efficiency": 0.8986386060714722, "metrics/particle/class_1/pid_accuracy": 0.9879437685012817, "metrics/particle/class_2/efficiency": 0.06172839552164078, "metrics/particle/class_2/pid_accuracy": 0.029411764815449715, "metrics/particle/class_3/efficiency": 0.4105607569217682, "metrics/particle/class_3/pid_accuracy": 0.9694218039512634, "metrics/particle/class_4/efficiency": 0.8979591727256775, "metrics/particle/class_4/pid_accuracy": 0.35606059432029724, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.44736841320991516, "metrics/particle/count/bias_mean": -21.28515625, "metrics/particle/count/mae": 21.29296875, "metrics/particle/count/prediction_mean": 58.28125, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3081044852733612, "metrics/particle/event/energy_response_mean": 0.6958687901496887, "metrics/particle/event/met_abs_error": 14.276457786560059, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25953686237335205, "metrics/particle/event/scalar_pt_response_mean": 0.7452413439750671, "metrics/particle/event/vector_pt_closure": 0.11271574348211288, "metrics/particle/matched/delta_eta_abs_mean": 0.007347016595304012, "metrics/particle/matched/delta_phi_abs_mean": 0.003971044439822435, "metrics/particle/matched/delta_r_mean": 0.009297134354710579, "metrics/particle/matched/energy_relative_abs_mean": 0.07917214930057526, "metrics/particle/matched/pid_accuracy": 0.9730836749076843, "metrics/particle/matched/pt_relative_abs_mean": 0.06654853373765945, "metrics/particle/matching/duplicate_fraction": 0.023815907537937164, "metrics/particle/matching/efficiency": 0.6511529088020325, "metrics/particle/matching/f1": 0.7516978979110718, "metrics/particle/matching/purity": 0.8889633417129517, "metrics/particle/matching/target_coverage_dr0p05": 0.6630827784538269, "metrics/particle/matching/target_coverage_dr0p10": 0.6787602305412292, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5772170424461365, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6644247174263}, "step_train_time": 1.4796037673950195, "step_valid_time": 9.846922035001626, "step_total_time": 14128.613849639893, "elapsed_training_loop_seconds": 14128.613849639893, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.801398721311166, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26719945124412836, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 596, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 516, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 465, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7435158501440923, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8657718120805369, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1342281879194631, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.670028818443804, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7802013422818792, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7209302325581395, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7435158501440923, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8556163031742232, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25696352775205633, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1461, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1338, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1241, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8699609882964889, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9158110882956879, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8922974324774925, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08418891170431211, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8068920676202861, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8494182067077344, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8276092030676893, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8699609882964889, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8159422009776844, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2763720471047997, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1073, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 961, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 872, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8342013888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8956197576887233, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8638202247191011, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1043802423112768, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7569444444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8126747437092264, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7838202247191012, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8342013888888888, "step_test_time": 21.75050414999896, "step_plot_time": 13.178170659999523, "step_validation_cycle_time": 45.27245032599967} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_52000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_52000.json new file mode 100644 index 0000000000000000000000000000000000000000..0a00518b3a7804461d22af672ae6926784089455 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_52000.json @@ -0,0 +1 @@ +{"epoch": 34.666666666666664, "train": {"Classification_binary": 0.24793587625026703, "Classification": 0.00032498803921043873, "Regression_pt": 0.2437341809272766, "Regression_eta": 0.033829715102910995, "Regression_sin_phi": 0.008926755748689175, "Regression_cos_phi": 0.009271553717553616, "Regression_energy": 0.25453832745552063, "Total": 2.179305076599121, "Total_unweighted": 0.7985613972414285}, "train_group": {}, "valid": {"Classification_binary": 0.24137655893961588, "Classification": 0.0003564356981466214, "Regression_pt": 0.26401861508687335, "Regression_eta": 0.04494906465212504, "Regression_sin_phi": 0.010743876298268637, "Regression_cos_phi": 0.01136434202392896, "Regression_energy": 0.2769702871640523, "Total": 2.588473637898763, "Total_unweighted": 0.8497791798630108, "metrics/particle/class_1/efficiency": 0.9055560827255249, "metrics/particle/class_1/pid_accuracy": 0.9883497953414917, "metrics/particle/class_2/efficiency": 0.060639072209596634, "metrics/particle/class_2/pid_accuracy": 0.0359281450510025, "metrics/particle/class_3/efficiency": 0.4323974847793579, "metrics/particle/class_3/pid_accuracy": 0.9690304398536682, "metrics/particle/class_4/efficiency": 0.918367326259613, "metrics/particle/class_4/pid_accuracy": 0.25925925374031067, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.429824560880661, "metrics/particle/count/bias_mean": -19.2890625, "metrics/particle/count/mae": 19.32291603088379, "metrics/particle/count/prediction_mean": 60.27734375, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3138587474822998, "metrics/particle/event/energy_response_mean": 0.6874468326568604, "metrics/particle/event/met_abs_error": 14.444092750549316, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2624282240867615, "metrics/particle/event/scalar_pt_response_mean": 0.7401148676872253, "metrics/particle/event/vector_pt_closure": 0.11310257762670517, "metrics/particle/matched/delta_eta_abs_mean": 0.007709278725087643, "metrics/particle/matched/delta_phi_abs_mean": 0.004369494039565325, "metrics/particle/matched/delta_r_mean": 0.009847838431596756, "metrics/particle/matched/energy_relative_abs_mean": 0.07775967568159103, "metrics/particle/matched/pid_accuracy": 0.9726014137268066, "metrics/particle/matched/pt_relative_abs_mean": 0.06513962149620056, "metrics/particle/matching/duplicate_fraction": 0.025252198800444603, "metrics/particle/matching/efficiency": 0.6641792058944702, "metrics/particle/matching/f1": 0.755791425704956, "metrics/particle/matching/purity": 0.8767200112342834, "metrics/particle/matching/target_coverage_dr0p05": 0.6723616123199463, "metrics/particle/matching/target_coverage_dr0p10": 0.6904773712158203, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.590668797492981, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6789238452911377}, "step_train_time": 1.4898250102996826, "step_valid_time": 9.699413025999092, "step_total_time": 14407.294603824615, "elapsed_training_loop_seconds": 14407.294603824615, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7982740860462277, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2643902379031995, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 594, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 466, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7463976945244957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8720538720538721, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8043478260869565, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12794612794612795, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6714697406340058, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7845117845117845, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7236024844720497, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7463976945244957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8539571451866489, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25282107679847576, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1466, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1327, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1232, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8628088426527959, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9051841746248295, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8834886817576565, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09481582537517053, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8010403120936281, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8403819918144612, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8202396804260985, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8628088426527959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8118095057209864, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28379575413554947, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 863, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8324652777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8896103896103896, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8600896860986547, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11038961038961038, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7491319444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.800556586270872, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7739910313901345, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8324652777777778, "step_test_time": 21.956665618001352, "step_plot_time": 13.16671678899911, "step_validation_cycle_time": 45.28942770700087} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_53000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_53000.json new file mode 100644 index 0000000000000000000000000000000000000000..5783905325fadd5d9e96a99107754d083cf301b7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_53000.json @@ -0,0 +1 @@ +{"epoch": 35.333333333333336, "train": {"Classification_binary": 0.23112910985946655, "Classification": 0.0003704705450218171, "Regression_pt": 0.2517852187156677, "Regression_eta": 0.025293942540884018, "Regression_sin_phi": 0.004452405031770468, "Regression_cos_phi": 0.0072178831323981285, "Regression_energy": 0.253254234790802, "Total": 1.665833592414856, "Total_unweighted": 0.7735032646160107}, "train_group": {}, "valid": {"Classification_binary": 0.24010934432347616, "Classification": 0.0003568242924908797, "Regression_pt": 0.26267621914545697, "Regression_eta": 0.04613527158896128, "Regression_sin_phi": 0.010922905057668686, "Regression_cos_phi": 0.011260284731785456, "Regression_energy": 0.27471540371576947, "Total": 2.6057167053222656, "Total_unweighted": 0.8461762528556089, "metrics/particle/class_1/efficiency": 0.9089674353599548, "metrics/particle/class_1/pid_accuracy": 0.98700350522995, "metrics/particle/class_2/efficiency": 0.06681191176176071, "metrics/particle/class_2/pid_accuracy": 0.027173912152647972, "metrics/particle/class_3/efficiency": 0.460359126329422, "metrics/particle/class_3/pid_accuracy": 0.9710767865180969, "metrics/particle/class_4/efficiency": 0.8877550959587097, "metrics/particle/class_4/pid_accuracy": 0.3448275923728943, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.48245614767074585, "metrics/particle/count/bias_mean": -18.34375, "metrics/particle/count/mae": 18.40364646911621, "metrics/particle/count/prediction_mean": 61.22265625, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.305096834897995, "metrics/particle/event/energy_response_mean": 0.700276792049408, "metrics/particle/event/met_abs_error": 14.735726356506348, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25350436568260193, "metrics/particle/event/scalar_pt_response_mean": 0.7529112696647644, "metrics/particle/event/vector_pt_closure": 0.11628282815217972, "metrics/particle/matched/delta_eta_abs_mean": 0.008059296756982803, "metrics/particle/matched/delta_phi_abs_mean": 0.003981837537139654, "metrics/particle/matched/delta_r_mean": 0.009938330389559269, "metrics/particle/matched/energy_relative_abs_mean": 0.07536736875772476, "metrics/particle/matched/pid_accuracy": 0.9726572632789612, "metrics/particle/matched/pt_relative_abs_mean": 0.06696581095457077, "metrics/particle/matching/duplicate_fraction": 0.025734277442097664, "metrics/particle/matching/efficiency": 0.6781055927276611, "metrics/particle/matching/f1": 0.7664576768875122, "metrics/particle/matching/purity": 0.8812820315361023, "metrics/particle/matching/target_coverage_dr0p05": 0.685682475566864, "metrics/particle/matching/target_coverage_dr0p10": 0.7032418251037598, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6018623113632202, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6928829550743103}, "step_train_time": 1.514526605606079, "step_valid_time": 9.667312040999605, "step_total_time": 14686.006245136261, "elapsed_training_loop_seconds": 14686.006245136261, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8142484603203122, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.260669419223148, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 596, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 471, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7521613832853026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8758389261744967, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8093023255813954, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12416107382550336, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6786743515850144, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.790268456375839, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7302325581395349, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7521613832853026, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.86936308001643, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24454796464684259, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1470, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1352, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1258, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8790637191157347, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9197278911564626, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.898936170212766, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08027210884353742, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8179453836150845, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.85578231292517, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.836436170212766, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8790637191157347, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8244877288251932, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2757057920395726, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1079, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 964, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 876, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8368055555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8934198331788693, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8641864634692963, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10658016682113068, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7604166666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8118628359592215, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.785298072613178, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8368055555555556, "step_test_time": 21.775129834000836, "step_plot_time": 13.110228200001075, "step_validation_cycle_time": 45.017957652000405} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_54000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_54000.json new file mode 100644 index 0000000000000000000000000000000000000000..e2e298560eb2e1b22cd6e4c66a95da114a34db3a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_54000.json @@ -0,0 +1 @@ +{"epoch": 36.0, "train": {"Classification_binary": 0.22911545634269714, "Classification": 0.0003024563193321228, "Regression_pt": 0.24673351645469666, "Regression_eta": 0.01553911529481411, "Regression_sin_phi": 0.010907486081123352, "Regression_cos_phi": 0.007134667597711086, "Regression_energy": 0.24087341129779816, "Total": 1.9094682931900024, "Total_unweighted": 0.7506061093881726}, "train_group": {}, "valid": {"Classification_binary": 0.24020602305730185, "Classification": 0.0003509159820775191, "Regression_pt": 0.26123444239298504, "Regression_eta": 0.043124149243036904, "Regression_sin_phi": 0.010566951086123785, "Regression_cos_phi": 0.011086021860440573, "Regression_energy": 0.27299753824869794, "Total": 2.530585289001465, "Total_unweighted": 0.8395660418706636, "metrics/particle/class_1/efficiency": 0.903376579284668, "metrics/particle/class_1/pid_accuracy": 0.9871328473091125, "metrics/particle/class_2/efficiency": 0.0671750158071518, "metrics/particle/class_2/pid_accuracy": 0.0810810774564743, "metrics/particle/class_3/efficiency": 0.44887012243270874, "metrics/particle/class_3/pid_accuracy": 0.9734723567962646, "metrics/particle/class_4/efficiency": 0.9047619104385376, "metrics/particle/class_4/pid_accuracy": 0.35338345170021057, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.34210526943206787, "metrics/particle/count/bias_mean": -19.21875, "metrics/particle/count/mae": 19.23177146911621, "metrics/particle/count/prediction_mean": 60.34765625, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.30007198452949524, "metrics/particle/event/energy_response_mean": 0.7017725110054016, "metrics/particle/event/met_abs_error": 14.131019592285156, "metrics/particle/event/scalar_pt_relative_abs_error": 0.24640750885009766, "metrics/particle/event/scalar_pt_response_mean": 0.7571110129356384, "metrics/particle/event/vector_pt_closure": 0.11175798624753952, "metrics/particle/matched/delta_eta_abs_mean": 0.007866257801651955, "metrics/particle/matched/delta_phi_abs_mean": 0.004043323919177055, "metrics/particle/matched/delta_r_mean": 0.009789454750716686, "metrics/particle/matched/energy_relative_abs_mean": 0.07970978319644928, "metrics/particle/matched/pid_accuracy": 0.9731959700584412, "metrics/particle/matched/pt_relative_abs_mean": 0.06881502270698547, "metrics/particle/matching/duplicate_fraction": 0.02548169158399105, "metrics/particle/matching/efficiency": 0.6703650951385498, "metrics/particle/matching/f1": 0.7624471783638, "metrics/particle/matching/purity": 0.8838543891906738, "metrics/particle/matching/target_coverage_dr0p05": 0.679774820804596, "metrics/particle/matching/target_coverage_dr0p10": 0.6963850259780884, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5922889113426208, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.684847891330719}, "step_train_time": 1.5602030754089355, "step_valid_time": 9.6938372730001, "step_total_time": 14964.287384033203, "elapsed_training_loop_seconds": 14964.287384033203, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8145191792075942, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2552813079392964, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 605, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 529, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 478, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7622478386167147, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8743801652892562, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8144726712856043, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1256198347107438, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6887608069164265, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7900826446280992, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7359507313317937, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7622478386167147, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8757526630065987, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24584284006377133, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1472, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1341, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1258, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8719115734720416, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9110054347826086, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.891029900332226, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08899456521739131, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8179453836150845, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8546195652173914, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8358803986710963, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8719115734720416, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8259138690385605, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27848013066134325, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1085, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 968, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 885, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8402777777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8921658986175115, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8654447921323201, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10783410138248847, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7682291666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.815668202764977, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7912382655341976, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8402777777777778, "step_test_time": 21.836349361999964, "step_plot_time": 13.766865227999006, "step_validation_cycle_time": 45.76141284099867} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_55000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_55000.json new file mode 100644 index 0000000000000000000000000000000000000000..8554463d0f91b238cd8cc450942549a6b0ad671b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_55000.json @@ -0,0 +1 @@ +{"epoch": 36.666666666666664, "train": {"Classification_binary": 0.24221116304397583, "Classification": 0.00033554676338098943, "Regression_pt": 0.2888014018535614, "Regression_eta": 0.030640047043561935, "Regression_sin_phi": 0.006827140226960182, "Regression_cos_phi": 0.006035156548023224, "Regression_energy": 0.30648303031921387, "Total": 1.8708373308181763, "Total_unweighted": 0.8813334857986774}, "train_group": {}, "valid": {"Classification_binary": 0.2415578564008077, "Classification": 0.0003576582142462333, "Regression_pt": 0.2585327625274658, "Regression_eta": 0.0461567888657252, "Regression_sin_phi": 0.010641492903232574, "Regression_cos_phi": 0.011234814922014872, "Regression_energy": 0.27178194125493366, "Total": 2.5842394828796387, "Total_unweighted": 0.840263315088426, "metrics/particle/class_1/efficiency": 0.9089042544364929, "metrics/particle/class_1/pid_accuracy": 0.9886707067489624, "metrics/particle/class_2/efficiency": 0.06935366988182068, "metrics/particle/class_2/pid_accuracy": 0.07329843193292618, "metrics/particle/class_3/efficiency": 0.4287833869457245, "metrics/particle/class_3/pid_accuracy": 0.9685919880867004, "metrics/particle/class_4/efficiency": 0.8911564350128174, "metrics/particle/class_4/pid_accuracy": 0.2557252049446106, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.35087719559669495, "metrics/particle/count/bias_mean": -19.21614646911621, "metrics/particle/count/mae": 19.24479103088379, "metrics/particle/count/prediction_mean": 60.35026168823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3061836063861847, "metrics/particle/event/energy_response_mean": 0.6963478922843933, "metrics/particle/event/met_abs_error": 14.190189361572266, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2491159439086914, "metrics/particle/event/scalar_pt_response_mean": 0.7543082237243652, "metrics/particle/event/vector_pt_closure": 0.11247880011796951, "metrics/particle/matched/delta_eta_abs_mean": 0.008177496492862701, "metrics/particle/matched/delta_phi_abs_mean": 0.004080728627741337, "metrics/particle/matched/delta_r_mean": 0.010104910470545292, "metrics/particle/matched/energy_relative_abs_mean": 0.07703416049480438, "metrics/particle/matched/pid_accuracy": 0.972274899482727, "metrics/particle/matched/pt_relative_abs_mean": 0.06734934449195862, "metrics/particle/matching/duplicate_fraction": 0.027853891253471375, "metrics/particle/matching/efficiency": 0.6646210551261902, "metrics/particle/matching/f1": 0.7559000849723816, "metrics/particle/matching/purity": 0.8762432932853699, "metrics/particle/matching/target_coverage_dr0p05": 0.6747344732284546, "metrics/particle/matching/target_coverage_dr0p10": 0.693062961101532, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5876413583755493, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6792675256729126}, "step_train_time": 1.53078031539917, "step_valid_time": 9.740336093000224, "step_total_time": 15243.737793207169, "elapsed_training_loop_seconds": 15243.737793207169, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.81719478856712, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2567753314954301, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 602, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 529, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 482, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7622478386167147, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8787375415282392, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.816358024691358, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1212624584717608, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6945244956772334, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8006644518272426, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7438271604938271, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7622478386167147, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8685486011495279, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2515268559667888, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1462, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1344, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1255, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8738621586475943, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9192886456908345, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.896, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08071135430916553, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8159947984395318, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8584131326949385, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8366666666666667, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8738621586475943, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8233434991176736, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.281766868589416, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1091, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 969, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 875, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8411458333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8881759853345554, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8640213999108337, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11182401466544455, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7595486111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8020164986251146, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.780205082478823, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8411458333333334, "step_test_time": 21.77030080499935, "step_plot_time": 13.204647862001366, "step_validation_cycle_time": 45.216648632000215} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_56000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_56000.json new file mode 100644 index 0000000000000000000000000000000000000000..07e362c4d6e642b49b959a01e73cbe4dfc91ae7c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_56000.json @@ -0,0 +1 @@ +{"epoch": 37.333333333333336, "train": {"Classification_binary": 0.24725660681724548, "Classification": 0.0002772974839899689, "Regression_pt": 0.31696587800979614, "Regression_eta": 0.02911163866519928, "Regression_sin_phi": 0.011634430848062038, "Regression_cos_phi": 0.00608190568163991, "Regression_energy": 0.3136204481124878, "Total": 2.166567802429199, "Total_unweighted": 0.9249482056184206}, "train_group": {}, "valid": {"Classification_binary": 0.23980887730916342, "Classification": 0.0003531110705807805, "Regression_pt": 0.2637095848719279, "Regression_eta": 0.04481680194536845, "Regression_sin_phi": 0.010829020291566849, "Regression_cos_phi": 0.01097786674896876, "Regression_energy": 0.2747420271237691, "Total": 2.5654242833455405, "Total_unweighted": 0.8452372893613451, "metrics/particle/class_1/efficiency": 0.9064404964447021, "metrics/particle/class_1/pid_accuracy": 0.9887444972991943, "metrics/particle/class_2/efficiency": 0.061002179980278015, "metrics/particle/class_2/pid_accuracy": 0.0357142873108387, "metrics/particle/class_3/efficiency": 0.4185878336429596, "metrics/particle/class_3/pid_accuracy": 0.9692810773849487, "metrics/particle/class_4/efficiency": 0.8979591727256775, "metrics/particle/class_4/pid_accuracy": 0.34090909361839294, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.3333333432674408, "metrics/particle/count/bias_mean": -20.62239646911621, "metrics/particle/count/mae": 20.63802146911621, "metrics/particle/count/prediction_mean": 58.94401168823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32049810886383057, "metrics/particle/event/energy_response_mean": 0.6823458671569824, "metrics/particle/event/met_abs_error": 14.233996391296387, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26317843794822693, "metrics/particle/event/scalar_pt_response_mean": 0.7398199439048767, "metrics/particle/event/vector_pt_closure": 0.11321970820426941, "metrics/particle/matched/delta_eta_abs_mean": 0.008709770627319813, "metrics/particle/matched/delta_phi_abs_mean": 0.003964944742619991, "metrics/particle/matched/delta_r_mean": 0.010505870915949345, "metrics/particle/matched/energy_relative_abs_mean": 0.07512148469686508, "metrics/particle/matched/pid_accuracy": 0.9733389616012573, "metrics/particle/matched/pt_relative_abs_mean": 0.06523750722408295, "metrics/particle/matching/duplicate_fraction": 0.02385738492012024, "metrics/particle/matching/efficiency": 0.658615231513977, "metrics/particle/matching/f1": 0.7566744089126587, "metrics/particle/matching/purity": 0.8890410661697388, "metrics/particle/matching/target_coverage_dr0p05": 0.6680412888526917, "metrics/particle/matching/target_coverage_dr0p10": 0.6847987771034241, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5854485034942627, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6723124980926514}, "step_train_time": 1.5159502029418945, "step_valid_time": 9.771548970000367, "step_total_time": 15523.341487646103, "elapsed_training_loop_seconds": 15523.341487646103, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8011894355904712, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2587525654045817, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 592, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 515, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 461, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7420749279538905, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8699324324324325, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8009331259720062, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13006756756756757, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6642651296829971, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7787162162162162, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7169517884914464, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7420749279538905, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8565747623453032, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24722864580333104, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1454, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1331, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1245, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8654096228868661, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9154057771664375, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8897058823529411, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08459422283356259, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8094928478543563, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8562585969738652, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8322192513368984, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8654096228868661, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8154939878693717, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2886952618008525, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1068, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 953, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 856, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8272569444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8923220973782772, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8585585585585586, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10767790262172285, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7430555555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8014981273408239, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7711711711711712, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8272569444444444, "step_test_time": 21.916759361000004, "step_plot_time": 13.096891866000078, "step_validation_cycle_time": 45.26199179499963} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_57000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_57000.json new file mode 100644 index 0000000000000000000000000000000000000000..0313e8309ee0a82cbd5243dccc97c60f31d08eb9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_57000.json @@ -0,0 +1 @@ +{"epoch": 38.0, "train": {"Classification_binary": 0.23244111239910126, "Classification": 0.00035156478406861424, "Regression_pt": 0.21853412687778473, "Regression_eta": 0.016073770821094513, "Regression_sin_phi": 0.004295943304896355, "Regression_cos_phi": 0.008390701375901699, "Regression_energy": 0.22486937046051025, "Total": 1.564452886581421, "Total_unweighted": 0.7049565900233574}, "train_group": {}, "valid": {"Classification_binary": 0.2392570972442627, "Classification": 0.00035126437433063984, "Regression_pt": 0.2611185312271118, "Regression_eta": 0.04605360825856527, "Regression_sin_phi": 0.010595152775446573, "Regression_cos_phi": 0.01096318413813909, "Regression_energy": 0.27389224370320636, "Total": 2.5628105799357095, "Total_unweighted": 0.8422310817210625, "metrics/particle/class_1/efficiency": 0.9060614705085754, "metrics/particle/class_1/pid_accuracy": 0.9895764589309692, "metrics/particle/class_2/efficiency": 0.061365287750959396, "metrics/particle/class_2/pid_accuracy": 0.047337278723716736, "metrics/particle/class_3/efficiency": 0.41303354501724243, "metrics/particle/class_3/pid_accuracy": 0.9684996008872986, "metrics/particle/class_4/efficiency": 0.8979591727256775, "metrics/particle/class_4/pid_accuracy": 0.2916666567325592, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.3245614171028137, "metrics/particle/count/bias_mean": -20.88020896911621, "metrics/particle/count/mae": 20.88802146911621, "metrics/particle/count/prediction_mean": 58.68619918823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3236514925956726, "metrics/particle/event/energy_response_mean": 0.6793692708015442, "metrics/particle/event/met_abs_error": 14.218235969543457, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2706605792045593, "metrics/particle/event/scalar_pt_response_mean": 0.7330134510993958, "metrics/particle/event/vector_pt_closure": 0.11265473812818527, "metrics/particle/matched/delta_eta_abs_mean": 0.008036418817937374, "metrics/particle/matched/delta_phi_abs_mean": 0.004254904109984636, "metrics/particle/matched/delta_r_mean": 0.010111748240888119, "metrics/particle/matched/energy_relative_abs_mean": 0.07614297419786453, "metrics/particle/matched/pid_accuracy": 0.9734091758728027, "metrics/particle/matched/pt_relative_abs_mean": 0.0666530430316925, "metrics/particle/matching/duplicate_fraction": 0.023562822490930557, "metrics/particle/matching/efficiency": 0.6560459733009338, "metrics/particle/matching/f1": 0.7551282048225403, "metrics/particle/matching/purity": 0.8894633054733276, "metrics/particle/matching/target_coverage_dr0p05": 0.6649647355079651, "metrics/particle/matching/target_coverage_dr0p10": 0.6812967658042908, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5840083956718445, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6692195534706116}, "step_train_time": 1.7378525733947754, "step_valid_time": 9.711222481999357, "step_total_time": 15801.574409008026, "elapsed_training_loop_seconds": 15801.574409008026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7893293061557032, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26831303089307335, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 596, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 519, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 460, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7478386167146974, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8708053691275168, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8046511627906977, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12919463087248323, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6628242074927954, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7718120805369127, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7131782945736435, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7478386167146974, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8468876339984467, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25774988165827073, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1467, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1343, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1246, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8732119635890767, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9154737559645535, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8938435940099834, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0845262440354465, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8101430429128739, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8493524199045671, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8292845257903494, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8732119635890767, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8025687836396768, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2782357043349838, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1077, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 965, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 867, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8376736111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8960074280408542, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8658591296545536, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10399257195914577, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7526041666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8050139275766016, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7779273216689099, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8376736111111112, "step_test_time": 21.677065738998863, "step_plot_time": 13.253041402000235, "step_validation_cycle_time": 45.10944738699982} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_58000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_58000.json new file mode 100644 index 0000000000000000000000000000000000000000..b2302f92c465b5bf4f3715b0b2987d67c627b2cd --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_58000.json @@ -0,0 +1 @@ +{"epoch": 38.666666666666664, "train": {"Classification_binary": 0.2123824506998062, "Classification": 0.00030288059497252107, "Regression_pt": 0.2780628204345703, "Regression_eta": 0.02305479347705841, "Regression_sin_phi": 0.008035887964069843, "Regression_cos_phi": 0.007312816567718983, "Regression_energy": 0.26792821288108826, "Total": 1.855255126953125, "Total_unweighted": 0.7970798626192845}, "train_group": {}, "valid": {"Classification_binary": 0.24069337050120035, "Classification": 0.0003546631196513772, "Regression_pt": 0.26773667335510254, "Regression_eta": 0.04574365417162577, "Regression_sin_phi": 0.011078921457131704, "Regression_cos_phi": 0.011305771768093109, "Regression_energy": 0.2807782292366028, "Total": 2.6197713216145835, "Total_unweighted": 0.8576912836094076, "metrics/particle/class_1/efficiency": 0.9022710919380188, "metrics/particle/class_1/pid_accuracy": 0.9886224269866943, "metrics/particle/class_2/efficiency": 0.05954974517226219, "metrics/particle/class_2/pid_accuracy": 0.05487804859876633, "metrics/particle/class_3/efficiency": 0.403522789478302, "metrics/particle/class_3/pid_accuracy": 0.9682285189628601, "metrics/particle/class_4/efficiency": 0.884353756904602, "metrics/particle/class_4/pid_accuracy": 0.3423076868057251, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.3070175349712372, "metrics/particle/count/bias_mean": -21.97395896911621, "metrics/particle/count/mae": 21.97916603088379, "metrics/particle/count/prediction_mean": 57.59244918823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31407496333122253, "metrics/particle/event/energy_response_mean": 0.688906729221344, "metrics/particle/event/met_abs_error": 14.058490753173828, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2636387050151825, "metrics/particle/event/scalar_pt_response_mean": 0.7394799590110779, "metrics/particle/event/vector_pt_closure": 0.11302268505096436, "metrics/particle/matched/delta_eta_abs_mean": 0.007652914617210627, "metrics/particle/matched/delta_phi_abs_mean": 0.003919568844139576, "metrics/particle/matched/delta_r_mean": 0.009541000239551067, "metrics/particle/matched/energy_relative_abs_mean": 0.08219645917415619, "metrics/particle/matched/pid_accuracy": 0.9731301665306091, "metrics/particle/matched/pt_relative_abs_mean": 0.0678115040063858, "metrics/particle/matching/duplicate_fraction": 0.02292509749531746, "metrics/particle/matching/efficiency": 0.649843692779541, "metrics/particle/matching/f1": 0.7539539337158203, "metrics/particle/matching/purity": 0.8977866172790527, "metrics/particle/matching/target_coverage_dr0p05": 0.6586479544639587, "metrics/particle/matching/target_coverage_dr0p10": 0.6748326420783997, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5766115188598633, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6613644957542419}, "step_train_time": 1.5366549491882324, "step_valid_time": 9.755500633000338, "step_total_time": 16081.02394080162, "elapsed_training_loop_seconds": 16081.02394080162, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7966960551019444, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2739363350178191, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 524, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 472, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7550432276657061, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8957264957264958, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8193901485535575, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10427350427350428, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6801152737752162, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8068376068376069, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7380766223612197, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7550432276657061, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.861180751099064, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25272588064181345, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1462, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1329, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1229, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.864109232769831, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9090287277701778, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.886, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09097127222982217, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7990897269180754, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8406292749658003, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8193333333333334, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.864109232769831, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8163560920199855, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28269750087176815, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1069, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 953, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 858, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8272569444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8914873713751169, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8581719945970284, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10851262862488306, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7447916666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8026192703461179, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7726249437190454, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8272569444444444, "step_test_time": 21.86051139800111, "step_plot_time": 13.192718940001214, "step_validation_cycle_time": 45.26386874199852} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_59000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_59000.json new file mode 100644 index 0000000000000000000000000000000000000000..825f00b64b71de51b6b9ceeb4ac613365ca402f8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_59000.json @@ -0,0 +1 @@ +{"epoch": 39.333333333333336, "train": {"Classification_binary": 0.24321140348911285, "Classification": 0.0002669779642019421, "Regression_pt": 0.2188555896282196, "Regression_eta": 0.020504074171185493, "Regression_sin_phi": 0.007386196404695511, "Regression_cos_phi": 0.006211449857801199, "Regression_energy": 0.24269022047519684, "Total": 1.6871861219406128, "Total_unweighted": 0.7391259119904134}, "train_group": {}, "valid": {"Classification_binary": 0.2394636869430542, "Classification": 0.0003520659326265256, "Regression_pt": 0.25908903280893963, "Regression_eta": 0.044260417421658836, "Regression_sin_phi": 0.01050615186492602, "Regression_cos_phi": 0.010776472588380178, "Regression_energy": 0.27095816532770794, "Total": 2.520402272542318, "Total_unweighted": 0.8354059928872934, "metrics/particle/class_1/efficiency": 0.90865159034729, "metrics/particle/class_1/pid_accuracy": 0.9877637624740601, "metrics/particle/class_2/efficiency": 0.06499636918306351, "metrics/particle/class_2/pid_accuracy": 0.06145251542329788, "metrics/particle/class_3/efficiency": 0.45160922408103943, "metrics/particle/class_3/pid_accuracy": 0.9721169471740723, "metrics/particle/class_4/efficiency": 0.918367326259613, "metrics/particle/class_4/pid_accuracy": 0.3037036955356598, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.45614033937454224, "metrics/particle/count/bias_mean": -18.81770896911621, "metrics/particle/count/mae": 18.84895896911621, "metrics/particle/count/prediction_mean": 60.74869918823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3034471571445465, "metrics/particle/event/energy_response_mean": 0.699489176273346, "metrics/particle/event/met_abs_error": 14.48178768157959, "metrics/particle/event/scalar_pt_relative_abs_error": 0.24980176985263824, "metrics/particle/event/scalar_pt_response_mean": 0.7544111609458923, "metrics/particle/event/vector_pt_closure": 0.11317526549100876, "metrics/particle/matched/delta_eta_abs_mean": 0.007363358978182077, "metrics/particle/matched/delta_phi_abs_mean": 0.004143914673477411, "metrics/particle/matched/delta_r_mean": 0.009421736001968384, "metrics/particle/matched/energy_relative_abs_mean": 0.07608089596033096, "metrics/particle/matched/pid_accuracy": 0.9732773303985596, "metrics/particle/matched/pt_relative_abs_mean": 0.06608673185110092, "metrics/particle/matching/duplicate_fraction": 0.025227734819054604, "metrics/particle/matching/efficiency": 0.6742435693740845, "metrics/particle/matching/f1": 0.7646665573120117, "metrics/particle/matching/purity": 0.8830993175506592, "metrics/particle/matching/target_coverage_dr0p05": 0.6843733191490173, "metrics/particle/matching/target_coverage_dr0p10": 0.7006071209907532, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5974602103233337, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6888408660888672}, "step_train_time": 1.5446105003356934, "step_valid_time": 9.708167484001024, "step_total_time": 16358.933441400528, "elapsed_training_loop_seconds": 16358.933441400528, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8212902318599601, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26996547824069717, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 590, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 524, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 478, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7550432276657061, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.888135593220339, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8161993769470405, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11186440677966102, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6887608069164265, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8101694915254237, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7445482866043613, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7550432276657061, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8805387629283822, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2424540184184265, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1470, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1351, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1271, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8784135240572172, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.919047619047619, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8982712765957447, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08095238095238096, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8263979193758128, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8646258503401361, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8450797872340425, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8784135240572172, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8284590024004972, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28109568304696597, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1075, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 970, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 885, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8420138888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9023255813953488, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8711270767849124, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09767441860465116, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7682291666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8232558139534883, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.794791198922317, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8420138888888888, "step_test_time": 21.74230453799828, "step_plot_time": 13.22381285099982, "step_validation_cycle_time": 45.22682032699959} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_6000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..8f9d27476ef96ea7a9dd95f03ef1e35e3486cfc7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 4.0, "train": {"Classification_binary": 0.2483118176460266, "Classification": 0.00043236519559286535, "Regression_pt": 0.3654688894748688, "Regression_eta": 0.023806113749742508, "Regression_sin_phi": 0.008832464925944805, "Regression_cos_phi": 0.01125653088092804, "Regression_energy": 0.37492334842681885, "Total": 2.3339462280273438, "Total_unweighted": 1.0330315302999225}, "train_group": {}, "valid": {"Classification_binary": 0.2702582875887553, "Classification": 0.0005050925149892768, "Regression_pt": 0.38800938924153644, "Regression_eta": 0.058175623416900635, "Regression_sin_phi": 0.013811897486448288, "Regression_cos_phi": 0.013781744986772537, "Regression_energy": 0.403248389561971, "Total": 3.313981374104818, "Total_unweighted": 1.1477904247973734, "metrics/particle/class_1/efficiency": 0.8757699131965637, "metrics/particle/class_1/pid_accuracy": 0.9864387512207031, "metrics/particle/class_2/efficiency": 0.05918663740158081, "metrics/particle/class_2/pid_accuracy": 0.0061349691823124886, "metrics/particle/class_3/efficiency": 0.32138779759407043, "metrics/particle/class_3/pid_accuracy": 0.9527698755264282, "metrics/particle/class_4/efficiency": 0.8333333134651184, "metrics/particle/class_4/pid_accuracy": 0.02448979578912258, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.1315789520740509, "metrics/particle/count/bias_mean": -24.40364646911621, "metrics/particle/count/mae": 24.44010353088379, "metrics/particle/count/prediction_mean": 55.16276168823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3264124393463135, "metrics/particle/event/energy_response_mean": 0.6760724186897278, "metrics/particle/event/met_abs_error": 15.200209617614746, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2800128757953644, "metrics/particle/event/scalar_pt_response_mean": 0.7238802313804626, "metrics/particle/event/vector_pt_closure": 0.11598166823387146, "metrics/particle/matched/delta_eta_abs_mean": 0.0088826147839427, "metrics/particle/matched/delta_phi_abs_mean": 0.003937539178878069, "metrics/particle/matched/delta_r_mean": 0.01061328686773777, "metrics/particle/matched/energy_relative_abs_mean": 0.08265917748212814, "metrics/particle/matched/pid_accuracy": 0.965255081653595, "metrics/particle/matched/pt_relative_abs_mean": 0.07331031560897827, "metrics/particle/matching/duplicate_fraction": 0.02374601736664772, "metrics/particle/matching/efficiency": 0.6005203723907471, "metrics/particle/matching/f1": 0.7092933058738708, "metrics/particle/matching/purity": 0.8661867380142212, "metrics/particle/matching/target_coverage_dr0p05": 0.6031714677810669, "metrics/particle/matching/target_coverage_dr0p10": 0.6255748271942139, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5391362905502319, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.620108962059021}, "step_train_time": 1.8891324996948242, "step_valid_time": 9.746021637000013, "step_total_time": 1631.1142053604126, "elapsed_training_loop_seconds": 1631.1142053604126, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8004725405017684, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2810170334576335, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 581, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 513, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 454, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.739193083573487, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.882960413080895, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8047058823529412, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11703958691910499, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.654178674351585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7814113597246127, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.712156862745098, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.739193083573487, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8387005459555461, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27379347465647297, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1452, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1312, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1192, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8530559167750326, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9035812672176309, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8775919732441472, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09641873278236915, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7750325097529259, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8209366391184573, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.797324414715719, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8530559167750326, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8006196496554312, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28191951476478583, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1063, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 941, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 817, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8168402777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8852304797742239, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8496613995485327, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1147695202257761, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7092013888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.768579492003763, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7376975169300226, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8168402777777778, "step_test_time": 20.776707133999935, "step_plot_time": 12.731360800000175, "step_validation_cycle_time": 43.68615243199997} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_60000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..5d11124f6080eb9ae59774e3bd308e3a1d45324d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 40.0, "train": {"Classification_binary": 0.2270384430885315, "Classification": 0.00044312761747278273, "Regression_pt": 0.24041101336479187, "Regression_eta": 0.008877543732523918, "Regression_sin_phi": 0.007654522079974413, "Regression_cos_phi": 0.010613568127155304, "Regression_energy": 0.2406870573759079, "Total": 1.8546411991119385, "Total_unweighted": 0.7357252753863577}, "train_group": {}, "valid": {"Classification_binary": 0.2400389313697815, "Classification": 0.00035322407105316717, "Regression_pt": 0.26099316279093426, "Regression_eta": 0.046002080043156944, "Regression_sin_phi": 0.010902094344298044, "Regression_cos_phi": 0.011686325073242188, "Regression_energy": 0.2736230492591858, "Total": 2.625003973642985, "Total_unweighted": 0.8435988669516519, "metrics/particle/class_1/efficiency": 0.9057772159576416, "metrics/particle/class_1/pid_accuracy": 0.989119827747345, "metrics/particle/class_2/efficiency": 0.061002179980278015, "metrics/particle/class_2/pid_accuracy": 0.0476190485060215, "metrics/particle/class_3/efficiency": 0.4126531183719635, "metrics/particle/class_3/pid_accuracy": 0.9692080616950989, "metrics/particle/class_4/efficiency": 0.8877550959587097, "metrics/particle/class_4/pid_accuracy": 0.3103448152542114, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.2631579041481018, "metrics/particle/count/bias_mean": -20.50650978088379, "metrics/particle/count/mae": 20.51171875, "metrics/particle/count/prediction_mean": 59.05989456176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31371548771858215, "metrics/particle/event/energy_response_mean": 0.6902822852134705, "metrics/particle/event/met_abs_error": 14.633532524108887, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2626417577266693, "metrics/particle/event/scalar_pt_response_mean": 0.7423162460327148, "metrics/particle/event/vector_pt_closure": 0.11630896478891373, "metrics/particle/matched/delta_eta_abs_mean": 0.008182154037058353, "metrics/particle/matched/delta_phi_abs_mean": 0.0038654382806271315, "metrics/particle/matched/delta_r_mean": 0.009998047724366188, "metrics/particle/matched/energy_relative_abs_mean": 0.07965314388275146, "metrics/particle/matched/pid_accuracy": 0.9732940793037415, "metrics/particle/matched/pt_relative_abs_mean": 0.06555930525064468, "metrics/particle/matching/duplicate_fraction": 0.025684554129838943, "metrics/particle/matching/efficiency": 0.6556695699691772, "metrics/particle/matching/f1": 0.7526605129241943, "metrics/particle/matching/purity": 0.8833281993865967, "metrics/particle/matching/target_coverage_dr0p05": 0.6684668064117432, "metrics/particle/matching/target_coverage_dr0p10": 0.684995174407959, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5801953673362732, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6688104271888733}, "step_train_time": 1.5153663158416748, "step_valid_time": 9.696813479997218, "step_total_time": 16638.155992507935, "elapsed_training_loop_seconds": 16638.155992507935, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8032151399071757, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25839033598121297, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 589, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 462, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7463976945244957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8794567062818336, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8074824629773967, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12054329371816638, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6657060518731989, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7843803056027164, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7201870615744349, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7463976945244957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8626105757512899, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24783196763258097, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1460, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1335, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1247, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8680104031209362, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9143835616438356, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8905937291527685, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08561643835616438, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8107932379713915, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8541095890410959, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8318879252835224, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8680104031209362, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.816692493104783, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28797676584436016, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1071, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 948, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 853, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8229166666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8851540616246498, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8529014844804319, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11484593837535013, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7404513888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7964519140989729, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7674313990103464, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8229166666666666, "step_test_time": 21.806277100000443, "step_plot_time": 13.250824244998512, "step_validation_cycle_time": 45.209734534000745} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_61000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_61000.json new file mode 100644 index 0000000000000000000000000000000000000000..4417949484dd7104d29036fe03054d064b64c47b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_61000.json @@ -0,0 +1 @@ +{"epoch": 40.666666666666664, "train": {"Classification_binary": 0.24326547980308533, "Classification": 0.0002498114772606641, "Regression_pt": 0.25501832365989685, "Regression_eta": 0.013114496134221554, "Regression_sin_phi": 0.006057362537831068, "Regression_cos_phi": 0.006325434893369675, "Regression_energy": 0.2722952365875244, "Total": 1.5490723848342896, "Total_unweighted": 0.7963261450931896}, "train_group": {}, "valid": {"Classification_binary": 0.23952319224675497, "Classification": 0.00034833059180527925, "Regression_pt": 0.25822410980860394, "Regression_eta": 0.046029855807622276, "Regression_sin_phi": 0.010688466330369314, "Regression_cos_phi": 0.01110591987768809, "Regression_energy": 0.27195386091868085, "Total": 2.573341210683187, "Total_unweighted": 0.8378737355815247, "metrics/particle/class_1/efficiency": 0.9047032594680786, "metrics/particle/class_1/pid_accuracy": 0.9906780123710632, "metrics/particle/class_2/efficiency": 0.06390704214572906, "metrics/particle/class_2/pid_accuracy": 0.07386363297700882, "metrics/particle/class_3/efficiency": 0.40968576073646545, "metrics/particle/class_3/pid_accuracy": 0.9703779220581055, "metrics/particle/class_4/efficiency": 0.9081632494926453, "metrics/particle/class_4/pid_accuracy": 0.28838950395584106, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.21052631735801697, "metrics/particle/count/bias_mean": -21.54166603088379, "metrics/particle/count/mae": 21.54947853088379, "metrics/particle/count/prediction_mean": 58.02473831176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.30503204464912415, "metrics/particle/event/energy_response_mean": 0.6988916397094727, "metrics/particle/event/met_abs_error": 13.925731658935547, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2507828176021576, "metrics/particle/event/scalar_pt_response_mean": 0.7528571486473083, "metrics/particle/event/vector_pt_closure": 0.11274059861898422, "metrics/particle/matched/delta_eta_abs_mean": 0.007394768297672272, "metrics/particle/matched/delta_phi_abs_mean": 0.003612290369346738, "metrics/particle/matched/delta_r_mean": 0.00910228118300438, "metrics/particle/matched/energy_relative_abs_mean": 0.07986818253993988, "metrics/particle/matched/pid_accuracy": 0.9742544293403625, "metrics/particle/matched/pt_relative_abs_mean": 0.06508009880781174, "metrics/particle/matching/duplicate_fraction": 0.02358458749949932, "metrics/particle/matching/efficiency": 0.6540658473968506, "metrics/particle/matching/f1": 0.7564682364463806, "metrics/particle/matching/purity": 0.8968875408172607, "metrics/particle/matching/target_coverage_dr0p05": 0.6659302711486816, "metrics/particle/matching/target_coverage_dr0p10": 0.6805276274681091, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5805063247680664, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6665030121803284}, "step_train_time": 1.549023151397705, "step_valid_time": 9.752102329999616, "step_total_time": 16916.95916748047, "elapsed_training_loop_seconds": 16916.95916748047, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8132444771944136, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2654189184990532, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 594, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 475, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7521613832853026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8787878787878788, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8105590062111802, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12121212121212122, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6844380403458213, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7996632996632996, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7375776397515528, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7521613832853026, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8756620009905146, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23899480768724235, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1462, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1339, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1263, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8706111833550065, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9158686730506156, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8926666666666667, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08413132694938441, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8211963589076723, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8638850889192886, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.842, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8706111833550065, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.819789177138579, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2871331771236112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1081, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 967, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 880, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8394097222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8945420906567992, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8660994178235557, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10545790934320073, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7638888888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8140610545790934, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7881773399014779, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8394097222222222, "step_test_time": 21.87977421600226, "step_plot_time": 13.286411016000784, "step_validation_cycle_time": 45.38424069300163} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_62000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_62000.json new file mode 100644 index 0000000000000000000000000000000000000000..8edabae6c8d436a7e8a20fb7525cc9905283cb6f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_62000.json @@ -0,0 +1 @@ +{"epoch": 41.333333333333336, "train": {"Classification_binary": 0.2552933990955353, "Classification": 0.00034228074946440756, "Regression_pt": 0.2185823917388916, "Regression_eta": 0.03986610472202301, "Regression_sin_phi": 0.005093188490718603, "Regression_cos_phi": 0.00632157176733017, "Regression_energy": 0.23354727029800415, "Total": 1.8277031183242798, "Total_unweighted": 0.7590462068619672}, "train_group": {}, "valid": {"Classification_binary": 0.23910011847813925, "Classification": 0.000350159980977575, "Regression_pt": 0.26314930121103924, "Regression_eta": 0.04730150600274404, "Regression_sin_phi": 0.011029845724503199, "Regression_cos_phi": 0.010633714497089386, "Regression_energy": 0.2787242929140727, "Total": 2.5906612078348794, "Total_unweighted": 0.8502889388085655, "metrics/particle/class_1/efficiency": 0.9082409143447876, "metrics/particle/class_1/pid_accuracy": 0.9878277778625488, "metrics/particle/class_2/efficiency": 0.0671750158071518, "metrics/particle/class_2/pid_accuracy": 0.0648648664355278, "metrics/particle/class_3/efficiency": 0.46058738231658936, "metrics/particle/class_3/pid_accuracy": 0.9719170928001404, "metrics/particle/class_4/efficiency": 0.8945578336715698, "metrics/particle/class_4/pid_accuracy": 0.35741445422172546, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.40707963705062866, "metrics/particle/count/bias_mean": -18.44400978088379, "metrics/particle/count/mae": 18.45182228088379, "metrics/particle/count/prediction_mean": 61.12239456176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.2930978834629059, "metrics/particle/event/energy_response_mean": 0.7128621935844421, "metrics/particle/event/met_abs_error": 13.84429931640625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.23641157150268555, "metrics/particle/event/scalar_pt_response_mean": 0.7675620913505554, "metrics/particle/event/vector_pt_closure": 0.11065006256103516, "metrics/particle/matched/delta_eta_abs_mean": 0.008241242729127407, "metrics/particle/matched/delta_phi_abs_mean": 0.004286974668502808, "metrics/particle/matched/delta_r_mean": 0.010294163599610329, "metrics/particle/matched/energy_relative_abs_mean": 0.07700079679489136, "metrics/particle/matched/pid_accuracy": 0.9734681844711304, "metrics/particle/matched/pt_relative_abs_mean": 0.0665188804268837, "metrics/particle/matching/duplicate_fraction": 0.02777896076440811, "metrics/particle/matching/efficiency": 0.6778601408004761, "metrics/particle/matching/f1": 0.7667261958122253, "metrics/particle/matching/purity": 0.882408082485199, "metrics/particle/matching/target_coverage_dr0p05": 0.6878262758255005, "metrics/particle/matching/target_coverage_dr0p10": 0.7049601674079895, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6007331609725952, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.692768394947052}, "step_train_time": 1.5237789154052734, "step_valid_time": 9.816914448998432, "step_total_time": 17195.57643055916, "elapsed_training_loop_seconds": 17195.57643055916, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8277632937532693, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2617420763427172, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 594, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 527, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 478, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7593659942363112, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8872053872053872, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8183229813664596, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1127946127946128, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6887608069164265, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8047138047138047, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7422360248447205, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7593659942363112, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8929311575147114, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23395094779798065, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1464, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1350, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1284, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8777633289986996, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9221311475409836, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.899400399733511, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0778688524590164, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.834850455136541, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8770491803278688, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8554297135243171, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8777633289986996, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8380962719103988, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27483398071218224, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1083, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 979, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 898, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8498263888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9039704524469068, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.876062639821029, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09602954755309326, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7795138888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8291782086795937, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8035794183445191, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8498263888888888, "step_test_time": 21.98948132499936, "step_plot_time": 13.321493649000331, "step_validation_cycle_time": 45.6025831199986} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_63000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_63000.json new file mode 100644 index 0000000000000000000000000000000000000000..cf7a6102fc23f4d85fb9b73d7713f68478c8a9ec --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_63000.json @@ -0,0 +1 @@ +{"epoch": 42.0, "train": {"Classification_binary": 0.24664562940597534, "Classification": 0.000418160721892491, "Regression_pt": 0.2847740650177002, "Regression_eta": 0.04229288548231125, "Regression_sin_phi": 0.010979852639138699, "Regression_cos_phi": 0.005833539180457592, "Regression_energy": 0.2947745621204376, "Total": 2.2888729572296143, "Total_unweighted": 0.8857186945679132}, "train_group": {}, "valid": {"Classification_binary": 0.24059462547302246, "Classification": 0.00035200431011617184, "Regression_pt": 0.2638370196024577, "Regression_eta": 0.046164890130360924, "Regression_sin_phi": 0.01138587420185407, "Regression_cos_phi": 0.011230902125438055, "Regression_energy": 0.27769704659779865, "Total": 2.635160446166992, "Total_unweighted": 0.8512623624410482, "metrics/particle/class_1/efficiency": 0.9067248106002808, "metrics/particle/class_1/pid_accuracy": 0.9870410561561584, "metrics/particle/class_2/efficiency": 0.068990558385849, "metrics/particle/class_2/pid_accuracy": 0.04736842215061188, "metrics/particle/class_3/efficiency": 0.4447234272956848, "metrics/particle/class_3/pid_accuracy": 0.9703165292739868, "metrics/particle/class_4/efficiency": 0.8911564350128174, "metrics/particle/class_4/pid_accuracy": 0.3129771053791046, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.4736842215061188, "metrics/particle/count/bias_mean": -18.31640625, "metrics/particle/count/mae": 18.34244728088379, "metrics/particle/count/prediction_mean": 61.25, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.2874719798564911, "metrics/particle/event/energy_response_mean": 0.7169312834739685, "metrics/particle/event/met_abs_error": 14.05153751373291, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2317143827676773, "metrics/particle/event/scalar_pt_response_mean": 0.7745099067687988, "metrics/particle/event/vector_pt_closure": 0.11446293443441391, "metrics/particle/matched/delta_eta_abs_mean": 0.008643378503620625, "metrics/particle/matched/delta_phi_abs_mean": 0.0041945199482142925, "metrics/particle/matched/delta_r_mean": 0.01061804499477148, "metrics/particle/matched/energy_relative_abs_mean": 0.07802512496709824, "metrics/particle/matched/pid_accuracy": 0.9721693396568298, "metrics/particle/matched/pt_relative_abs_mean": 0.06934085488319397, "metrics/particle/matching/duplicate_fraction": 0.030335884541273117, "metrics/particle/matching/efficiency": 0.6703323721885681, "metrics/particle/matching/f1": 0.7575244903564453, "metrics/particle/matching/purity": 0.8707908391952515, "metrics/particle/matching/target_coverage_dr0p05": 0.6812639832496643, "metrics/particle/matching/target_coverage_dr0p10": 0.7010162472724915, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5912907123565674, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6868771314620972}, "step_train_time": 1.5500173568725586, "step_valid_time": 9.804757571000664, "step_total_time": 17475.337914466858, "elapsed_training_loop_seconds": 17475.337914466858, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8394584001338399, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2676002989255716, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 600, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 524, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 481, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7550432276657061, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8733333333333333, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8098918083462133, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12666666666666668, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6930835734870316, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8016666666666666, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7434312210200927, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7550432276657061, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8849051001479247, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23385068285674382, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1479, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1351, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1275, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8784135240572172, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9134550371872887, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8955916473317865, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08654496281271129, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8289986996098829, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8620689655172413, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8452104739807756, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8784135240572172, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8478380894811719, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2762828975057994, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1086, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 978, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 896, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8489583333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9005524861878453, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8739946380697051, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09944751381215469, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7777777777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8250460405156538, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8007149240393209, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8489583333333334, "step_test_time": 21.85931206099849, "step_plot_time": 13.313687607002066, "step_validation_cycle_time": 45.445324911997886} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_64000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_64000.json new file mode 100644 index 0000000000000000000000000000000000000000..ffe6a24baec604c11f478e4e39a5d1835c1c7269 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_64000.json @@ -0,0 +1 @@ +{"epoch": 42.666666666666664, "train": {"Classification_binary": 0.23280219733715057, "Classification": 0.0003432254306972027, "Regression_pt": 0.2907663881778717, "Regression_eta": 0.014687817543745041, "Regression_sin_phi": 0.004810640588402748, "Regression_cos_phi": 0.005478138569742441, "Regression_energy": 0.27744585275650024, "Total": 1.480499267578125, "Total_unweighted": 0.82633426040411}, "train_group": {}, "valid": {"Classification_binary": 0.2390029033025106, "Classification": 0.00034370459616184235, "Regression_pt": 0.2547614773114522, "Regression_eta": 0.04610219101111094, "Regression_sin_phi": 0.01104588434100151, "Regression_cos_phi": 0.011280295749505361, "Regression_energy": 0.2673233548800151, "Total": 2.600396474202474, "Total_unweighted": 0.8298598111917574, "metrics/particle/class_1/efficiency": 0.9079882502555847, "metrics/particle/class_1/pid_accuracy": 0.9893550276756287, "metrics/particle/class_2/efficiency": 0.06463325768709183, "metrics/particle/class_2/pid_accuracy": 0.08426966518163681, "metrics/particle/class_3/efficiency": 0.4276040494441986, "metrics/particle/class_3/pid_accuracy": 0.9700177907943726, "metrics/particle/class_4/efficiency": 0.8979591727256775, "metrics/particle/class_4/pid_accuracy": 0.3068181872367859, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.44247788190841675, "metrics/particle/count/bias_mean": -20.76953125, "metrics/particle/count/mae": 20.79036521911621, "metrics/particle/count/prediction_mean": 58.796875, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3130457103252411, "metrics/particle/event/energy_response_mean": 0.692141592502594, "metrics/particle/event/met_abs_error": 14.044150352478027, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26218751072883606, "metrics/particle/event/scalar_pt_response_mean": 0.7399423122406006, "metrics/particle/event/vector_pt_closure": 0.10997676104307175, "metrics/particle/matched/delta_eta_abs_mean": 0.007670272141695023, "metrics/particle/matched/delta_phi_abs_mean": 0.004069935530424118, "metrics/particle/matched/delta_r_mean": 0.009642560966312885, "metrics/particle/matched/energy_relative_abs_mean": 0.07620970904827118, "metrics/particle/matched/pid_accuracy": 0.9740509390830994, "metrics/particle/matched/pt_relative_abs_mean": 0.06450022011995316, "metrics/particle/matching/duplicate_fraction": 0.021481087431311607, "metrics/particle/matching/efficiency": 0.6634427905082703, "metrics/particle/matching/f1": 0.7630313634872437, "metrics/particle/matching/purity": 0.8977987170219421, "metrics/particle/matching/target_coverage_dr0p05": 0.668843150138855, "metrics/particle/matching/target_coverage_dr0p10": 0.6846514940261841, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5902924537658691, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6765182614326477}, "step_train_time": 1.5445551872253418, "step_valid_time": 9.793249926999124, "step_total_time": 17754.01829957962, "elapsed_training_loop_seconds": 17754.01829957962, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7893241100943014, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26512187719102975, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 596, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 521, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7507204610951008, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8741610738255033, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8077519379844961, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12583892617449666, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6671469740634006, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7768456375838926, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7178294573643411, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7507204610951008, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8590573876616299, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24805630849336768, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1457, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1333, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1244, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8667100130039012, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9148936170212766, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8901502504173623, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0851063829787234, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8088426527958388, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.853809196980096, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8307178631051753, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8667100130039012, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8066681575258602, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27629431784431957, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1072, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 967, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 873, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8394097222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9020522388059702, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8696043165467626, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09794776119402986, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7578125, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8143656716417911, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7850719424460432, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8394097222222222, "step_test_time": 21.940799555999547, "step_plot_time": 13.26693777699984, "step_validation_cycle_time": 45.47363396200308} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_65000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_65000.json new file mode 100644 index 0000000000000000000000000000000000000000..2eeadabcbc529027dc7edc7b3cb8a4200b01aa46 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_65000.json @@ -0,0 +1 @@ +{"epoch": 43.333333333333336, "train": {"Classification_binary": 0.23011547327041626, "Classification": 0.0003381146234460175, "Regression_pt": 0.29621267318725586, "Regression_eta": 0.035130713135004044, "Regression_sin_phi": 0.0053963977843523026, "Regression_cos_phi": 0.00805845856666565, "Regression_energy": 0.3006848692893982, "Total": 1.9476020336151123, "Total_unweighted": 0.8759366998565383}, "train_group": {}, "valid": {"Classification_binary": 0.23879067103068033, "Classification": 0.00034725607838481665, "Regression_pt": 0.2569829821586609, "Regression_eta": 0.045804073413213096, "Regression_sin_phi": 0.010557368397712708, "Regression_cos_phi": 0.010648444294929504, "Regression_energy": 0.2684226830800374, "Total": 2.5314087867736816, "Total_unweighted": 0.8315534784536187, "metrics/particle/class_1/efficiency": 0.912189245223999, "metrics/particle/class_1/pid_accuracy": 0.9908238053321838, "metrics/particle/class_2/efficiency": 0.06209150329232216, "metrics/particle/class_2/pid_accuracy": 0.05263157933950424, "metrics/particle/class_3/efficiency": 0.41653352975845337, "metrics/particle/class_3/pid_accuracy": 0.9711388945579529, "metrics/particle/class_4/efficiency": 0.9047619104385376, "metrics/particle/class_4/pid_accuracy": 0.2518796920776367, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.35087719559669495, "metrics/particle/count/bias_mean": -20.75390625, "metrics/particle/count/mae": 20.75650978088379, "metrics/particle/count/prediction_mean": 58.8125, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3110930025577545, "metrics/particle/event/energy_response_mean": 0.6967421174049377, "metrics/particle/event/met_abs_error": 14.794551849365234, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2598835825920105, "metrics/particle/event/scalar_pt_response_mean": 0.7459402680397034, "metrics/particle/event/vector_pt_closure": 0.11703065782785416, "metrics/particle/matched/delta_eta_abs_mean": 0.007828690111637115, "metrics/particle/matched/delta_phi_abs_mean": 0.003967184107750654, "metrics/particle/matched/delta_r_mean": 0.009723134338855743, "metrics/particle/matched/energy_relative_abs_mean": 0.07439430058002472, "metrics/particle/matched/pid_accuracy": 0.9748384356498718, "metrics/particle/matched/pt_relative_abs_mean": 0.06315132975578308, "metrics/particle/matching/duplicate_fraction": 0.023423662409186363, "metrics/particle/matching/efficiency": 0.6607917547225952, "metrics/particle/matching/f1": 0.7598965167999268, "metrics/particle/matching/purity": 0.8939735889434814, "metrics/particle/matching/target_coverage_dr0p05": 0.6701686978340149, "metrics/particle/matching/target_coverage_dr0p10": 0.6849787831306458, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5889341831207275, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.673916220664978}, "step_train_time": 1.7446231842041016, "step_valid_time": 9.73134656700131, "step_total_time": 18033.930875062943, "elapsed_training_loop_seconds": 18033.930875062943, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8036603698290876, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26704883756225284, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 594, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 517, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7449567723342939, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8703703703703703, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8027950310559007, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12962962962962962, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6671469740634006, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7794612794612794, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.718944099378882, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7449567723342939, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8559419449221297, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26131018782213455, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1459, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1339, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1248, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8706111833550065, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9177518848526388, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8935602268935602, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0822481151473612, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.811443433029909, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8553803975325566, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8328328328328328, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8706111833550065, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8121015778278725, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2897309255752373, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1081, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 968, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 867, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8402777777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8954671600370028, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8669950738916257, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10453283996299723, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7526041666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8020351526364478, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7765338110165696, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8402777777777778, "step_test_time": 21.735172346998297, "step_plot_time": 13.409315938999498, "step_validation_cycle_time": 45.419971905997954} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_66000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_66000.json new file mode 100644 index 0000000000000000000000000000000000000000..200ce649dad0956208f17474b6534c3e34b2a3a2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_66000.json @@ -0,0 +1 @@ +{"epoch": 44.0, "train": {"Classification_binary": 0.25297895073890686, "Classification": 0.0003643831587396562, "Regression_pt": 0.27057594060897827, "Regression_eta": 0.013183506205677986, "Regression_sin_phi": 0.01148733589798212, "Regression_cos_phi": 0.007158395368605852, "Regression_energy": 0.28080058097839355, "Total": 1.9954463243484497, "Total_unweighted": 0.8365490929572843}, "train_group": {}, "valid": {"Classification_binary": 0.23975561062494913, "Classification": 0.00035724831589808065, "Regression_pt": 0.2618543306986491, "Regression_eta": 0.04511803388595581, "Regression_sin_phi": 0.011416309823592504, "Regression_cos_phi": 0.011337026953697205, "Regression_energy": 0.2745327552159627, "Total": 2.62720521291097, "Total_unweighted": 0.8443713155187047, "metrics/particle/class_1/efficiency": 0.9114627838134766, "metrics/particle/class_1/pid_accuracy": 0.988148033618927, "metrics/particle/class_2/efficiency": 0.06572258472442627, "metrics/particle/class_2/pid_accuracy": 0.027624309062957764, "metrics/particle/class_3/efficiency": 0.45195162296295166, "metrics/particle/class_3/pid_accuracy": 0.9750841856002808, "metrics/particle/class_4/efficiency": 0.9115646481513977, "metrics/particle/class_4/pid_accuracy": 0.33208954334259033, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.35087719559669495, "metrics/particle/count/bias_mean": -19.38150978088379, "metrics/particle/count/mae": 19.39453125, "metrics/particle/count/prediction_mean": 60.18489456176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.30983349680900574, "metrics/particle/event/energy_response_mean": 0.6929192543029785, "metrics/particle/event/met_abs_error": 14.055630683898926, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2560529112815857, "metrics/particle/event/scalar_pt_response_mean": 0.7479414939880371, "metrics/particle/event/vector_pt_closure": 0.11115812510251999, "metrics/particle/matched/delta_eta_abs_mean": 0.007482059765607119, "metrics/particle/matched/delta_phi_abs_mean": 0.003836374031379819, "metrics/particle/matched/delta_r_mean": 0.00932588241994381, "metrics/particle/matched/energy_relative_abs_mean": 0.07943108677864075, "metrics/particle/matched/pid_accuracy": 0.9741640090942383, "metrics/particle/matched/pt_relative_abs_mean": 0.06487256288528442, "metrics/particle/matching/duplicate_fraction": 0.025464065372943878, "metrics/particle/matching/efficiency": 0.6758472919464111, "metrics/particle/matching/f1": 0.76957768201828, "metrics/particle/matching/purity": 0.8934922814369202, "metrics/particle/matching/target_coverage_dr0p05": 0.6815585494041443, "metrics/particle/matching/target_coverage_dr0p10": 0.6973996162414551, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6025332808494568, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6904446482658386}, "step_train_time": 1.7828550338745117, "step_valid_time": 9.744834050998179, "step_total_time": 18313.019327163696, "elapsed_training_loop_seconds": 18313.019327163696, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8021758445111151, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2702225429547719, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 582, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 526, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 474, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7579250720461095, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9037800687285223, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8244514106583072, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09621993127147767, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6829971181556196, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8144329896907216, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7429467084639498, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7579250720461095, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8715380462487946, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24349018010777823, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1471, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1348, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1271, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8764629388816645, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9163834126444595, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8959787304752409, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08361658735554045, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8263979193758128, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8640380693405847, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.844798936523762, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8764629388816645, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.82768024589076, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2705142561296282, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1075, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 961, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 874, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8342013888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.893953488372093, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8630444544229906, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10604651162790697, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7586805555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8130232558139535, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7849124382577458, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8342013888888888, "step_test_time": 22.077345882000373, "step_plot_time": 13.20496057299897, "step_validation_cycle_time": 45.5018319539995} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_67000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_67000.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9f7a0efabb89c0f6c32bbac950b51505e0567f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_67000.json @@ -0,0 +1 @@ +{"epoch": 44.666666666666664, "train": {"Classification_binary": 0.2104009985923767, "Classification": 0.00023247289936989546, "Regression_pt": 0.18173259496688843, "Regression_eta": 0.01188726257532835, "Regression_sin_phi": 0.004097783472388983, "Regression_cos_phi": 0.003421780886128545, "Regression_energy": 0.1884060949087143, "Total": 1.107716679573059, "Total_unweighted": 0.6001789883011952}, "train_group": {}, "valid": {"Classification_binary": 0.23928993940353394, "Classification": 0.00035449703379223746, "Regression_pt": 0.25608036915461224, "Regression_eta": 0.043953085939089455, "Regression_sin_phi": 0.01086930309732755, "Regression_cos_phi": 0.010925985872745514, "Regression_energy": 0.2684391339619954, "Total": 2.5452095667521157, "Total_unweighted": 0.8299123144630964, "metrics/particle/class_1/efficiency": 0.9079566597938538, "metrics/particle/class_1/pid_accuracy": 0.9906418323516846, "metrics/particle/class_2/efficiency": 0.06572258472442627, "metrics/particle/class_2/pid_accuracy": 0.07182320207357407, "metrics/particle/class_3/efficiency": 0.42410409450531006, "metrics/particle/class_3/pid_accuracy": 0.9722820520401001, "metrics/particle/class_4/efficiency": 0.8945578336715698, "metrics/particle/class_4/pid_accuracy": 0.2775665521621704, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.31578946113586426, "metrics/particle/count/bias_mean": -20.76822853088379, "metrics/particle/count/mae": 20.78125, "metrics/particle/count/prediction_mean": 58.79817581176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3094043433666229, "metrics/particle/event/energy_response_mean": 0.6952764391899109, "metrics/particle/event/met_abs_error": 14.301432609558105, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2606427073478699, "metrics/particle/event/scalar_pt_response_mean": 0.7438222765922546, "metrics/particle/event/vector_pt_closure": 0.11265731602907181, "metrics/particle/matched/delta_eta_abs_mean": 0.00837012380361557, "metrics/particle/matched/delta_phi_abs_mean": 0.0036709075793623924, "metrics/particle/matched/delta_r_mean": 0.010049063712358475, "metrics/particle/matched/energy_relative_abs_mean": 0.07647877186536789, "metrics/particle/matched/pid_accuracy": 0.9749326109886169, "metrics/particle/matched/pt_relative_abs_mean": 0.06321173161268234, "metrics/particle/matching/duplicate_fraction": 0.022875744849443436, "metrics/particle/matching/efficiency": 0.6619699597358704, "metrics/particle/matching/f1": 0.761330246925354, "metrics/particle/matching/purity": 0.8957858085632324, "metrics/particle/matching/target_coverage_dr0p05": 0.668695867061615, "metrics/particle/matching/target_coverage_dr0p10": 0.6848970055580139, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5906524658203125, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6748654246330261}, "step_train_time": 1.6834652423858643, "step_valid_time": 9.722570353002084, "step_total_time": 18591.497109889984, "elapsed_training_loop_seconds": 18591.497109889984, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8014339506935754, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2709905315815154, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 591, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 473, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7521613832853026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.883248730964467, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8124513618677043, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.116751269035533, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6815561959654178, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8003384094754653, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7361867704280156, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7521613832853026, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8609696513928791, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2508758196687475, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1461, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1334, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1237, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8673602080624188, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9130732375085557, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8896298766255418, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08692676249144421, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8042912873862159, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8466803559206023, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8249416472157386, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8673602080624188, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8130329073531917, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2822150643886935, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1074, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 970, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 879, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8420138888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9031657355679702, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.87151841868823, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09683426443202979, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7630208333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8184357541899442, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7897574123989218, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8420138888888888, "step_test_time": 21.8397357319991, "step_plot_time": 13.117762461002712, "step_validation_cycle_time": 45.1477868740003} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_68000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_68000.json new file mode 100644 index 0000000000000000000000000000000000000000..ad08989467b18c71acf145948a2bd8c4a99097d8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_68000.json @@ -0,0 +1 @@ +{"epoch": 45.333333333333336, "train": {"Classification_binary": 0.23966945707798004, "Classification": 0.00035257069976069033, "Regression_pt": 0.22868527472019196, "Regression_eta": 0.02790408954024315, "Regression_sin_phi": 0.007646315731108189, "Regression_cos_phi": 0.004966472275555134, "Regression_energy": 0.24157123267650604, "Total": 1.751285195350647, "Total_unweighted": 0.7507954127213452}, "train_group": {}, "valid": {"Classification_binary": 0.239697535832723, "Classification": 0.0003483622955779235, "Regression_pt": 0.2627950112024943, "Regression_eta": 0.04447969297568003, "Regression_sin_phi": 0.01064301903049151, "Regression_cos_phi": 0.011416922012964884, "Regression_energy": 0.27631638447443646, "Total": 2.5741424560546875, "Total_unweighted": 0.845696927824368, "metrics/particle/class_1/efficiency": 0.9075460433959961, "metrics/particle/class_1/pid_accuracy": 0.9891758561134338, "metrics/particle/class_2/efficiency": 0.06790123134851456, "metrics/particle/class_2/pid_accuracy": 0.06417112052440643, "metrics/particle/class_3/efficiency": 0.3988054394721985, "metrics/particle/class_3/pid_accuracy": 0.9659448862075806, "metrics/particle/class_4/efficiency": 0.884353756904602, "metrics/particle/class_4/pid_accuracy": 0.41923075914382935, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.2982456088066101, "metrics/particle/count/bias_mean": -21.203125, "metrics/particle/count/mae": 21.20572853088379, "metrics/particle/count/prediction_mean": 58.36328125, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.30675962567329407, "metrics/particle/event/energy_response_mean": 0.7004539966583252, "metrics/particle/event/met_abs_error": 14.442301750183105, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25580963492393494, "metrics/particle/event/scalar_pt_response_mean": 0.7506783604621887, "metrics/particle/event/vector_pt_closure": 0.11219310015439987, "metrics/particle/matched/delta_eta_abs_mean": 0.007625749334692955, "metrics/particle/matched/delta_phi_abs_mean": 0.004115140996873379, "metrics/particle/matched/delta_r_mean": 0.009633959271013737, "metrics/particle/matched/energy_relative_abs_mean": 0.07609742134809494, "metrics/particle/matched/pid_accuracy": 0.9729987978935242, "metrics/particle/matched/pt_relative_abs_mean": 0.06454082578420639, "metrics/particle/matching/duplicate_fraction": 0.022176114842295647, "metrics/particle/matching/efficiency": 0.6509237885475159, "metrics/particle/matching/f1": 0.7509865164756775, "metrics/particle/matching/purity": 0.8874015808105469, "metrics/particle/matching/target_coverage_dr0p05": 0.659891664981842, "metrics/particle/matching/target_coverage_dr0p10": 0.6771891713142395, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5775279402732849, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6645392775535583}, "step_train_time": 1.4337852001190186, "step_valid_time": 9.800959432999662, "step_total_time": 18870.944570541382, "elapsed_training_loop_seconds": 18870.944570541382, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8110851607059428, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26323461507029255, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 599, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 523, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 467, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7536023054755043, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8731218697829716, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8089713843774169, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12687813021702837, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6729106628242075, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7796327212020033, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7223511214230471, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7536023054755043, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8584883437064261, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.244321496095931, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1459, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1336, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1247, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8686605981794538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9156956819739548, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8915582248915582, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08430431802604524, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8107932379713915, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8546949965729952, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8321654988321655, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8686605981794538, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.817053535159179, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27838985845543684, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1081, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 980, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 887, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8506944444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.906567992599445, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.877742946708464, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09343200740055504, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7699652777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.820536540240518, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7944469323779668, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8506944444444444, "step_test_time": 21.84386869300215, "step_plot_time": 13.289883241999632, "step_validation_cycle_time": 45.40535978000116} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_69000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_69000.json new file mode 100644 index 0000000000000000000000000000000000000000..1c6a1117875e52b1cedb4ee4251689ac0acfdef4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_69000.json @@ -0,0 +1 @@ +{"epoch": 46.0, "train": {"Classification_binary": 0.23846851289272308, "Classification": 0.00031011246028356254, "Regression_pt": 0.23797456920146942, "Regression_eta": 0.016504649072885513, "Regression_sin_phi": 0.006237404886633158, "Regression_cos_phi": 0.005688153672963381, "Regression_energy": 0.2264707386493683, "Total": 1.540842056274414, "Total_unweighted": 0.7316541408363264}, "train_group": {}, "valid": {"Classification_binary": 0.23901355266571045, "Classification": 0.00035253042976061505, "Regression_pt": 0.2556867003440857, "Regression_eta": 0.04244531194368998, "Regression_sin_phi": 0.010863512754440308, "Regression_cos_phi": 0.011073607951402664, "Regression_energy": 0.26883349816004437, "Total": 2.5331246058146157, "Total_unweighted": 0.8282687142491341, "metrics/particle/class_1/efficiency": 0.9082409143447876, "metrics/particle/class_1/pid_accuracy": 0.9888363480567932, "metrics/particle/class_2/efficiency": 0.06463325768709183, "metrics/particle/class_2/pid_accuracy": 0.03932584449648857, "metrics/particle/class_3/efficiency": 0.4339572489261627, "metrics/particle/class_3/pid_accuracy": 0.9723854064941406, "metrics/particle/class_4/efficiency": 0.8911564350128174, "metrics/particle/class_4/pid_accuracy": 0.3435114622116089, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.32743361592292786, "metrics/particle/count/bias_mean": -19.92708396911621, "metrics/particle/count/mae": 19.94010353088379, "metrics/particle/count/prediction_mean": 59.63932418823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31872594356536865, "metrics/particle/event/energy_response_mean": 0.6837451457977295, "metrics/particle/event/met_abs_error": 13.990486145019531, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26623931527137756, "metrics/particle/event/scalar_pt_response_mean": 0.7363497614860535, "metrics/particle/event/vector_pt_closure": 0.11185315251350403, "metrics/particle/matched/delta_eta_abs_mean": 0.008196915499866009, "metrics/particle/matched/delta_phi_abs_mean": 0.003752012038603425, "metrics/particle/matched/delta_r_mean": 0.00992616917937994, "metrics/particle/matched/energy_relative_abs_mean": 0.07696686685085297, "metrics/particle/matched/pid_accuracy": 0.9740875363349915, "metrics/particle/matched/pt_relative_abs_mean": 0.06561866402626038, "metrics/particle/matching/duplicate_fraction": 0.024081392213702202, "metrics/particle/matching/efficiency": 0.6662738919258118, "metrics/particle/matching/f1": 0.7616499662399292, "metrics/particle/matching/purity": 0.888893723487854, "metrics/particle/matching/target_coverage_dr0p05": 0.6743744611740112, "metrics/particle/matching/target_coverage_dr0p10": 0.6904937028884888, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5905542969703674, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6802330613136292}, "step_train_time": 1.4599850177764893, "step_valid_time": 9.768826356001227, "step_total_time": 19149.87375688553, "elapsed_training_loop_seconds": 19149.87375688553, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8006015851786605, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2681157850058229, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 594, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 473, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7521613832853026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8787878787878788, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8105590062111802, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12121212121212122, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6815561959654178, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7962962962962963, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.734472049689441, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7521613832853026, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8492727083958626, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24476715218234302, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1452, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1329, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1236, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.864109232769831, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9152892561983471, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8889632107023412, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08471074380165289, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8036410923276983, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8512396694214877, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8267558528428094, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.864109232769831, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8070993000262012, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28082891273716504, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1075, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 970, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 871, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8420138888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9023255813953488, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8711270767849124, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09767441860465116, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7560763888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8102325581395349, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7822182308037718, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8420138888888888, "step_test_time": 22.28612366300149, "step_plot_time": 13.266361240999686, "step_validation_cycle_time": 45.81667703099811} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_7000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..f4ac9730575c2c355a2d828092a95758727e1079 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 4.666666666666667, "train": {"Classification_binary": 0.28546369075775146, "Classification": 0.0006659476202912629, "Regression_pt": 0.42495641112327576, "Regression_eta": 0.036176007241010666, "Regression_sin_phi": 0.015153704211115837, "Regression_cos_phi": 0.01912013627588749, "Regression_energy": 0.45178455114364624, "Total": 3.5359115600585938, "Total_unweighted": 1.2333204483729787}, "train_group": {}, "valid": {"Classification_binary": 0.26777011156082153, "Classification": 0.0004933382927750548, "Regression_pt": 0.37778957684834796, "Regression_eta": 0.05684153735637665, "Regression_sin_phi": 0.013550789405902227, "Regression_cos_phi": 0.013579891373713812, "Regression_energy": 0.39407650629679364, "Total": 3.2511107126871743, "Total_unweighted": 1.1241017511347309, "metrics/particle/class_1/efficiency": 0.8753277063369751, "metrics/particle/class_1/pid_accuracy": 0.986359715461731, "metrics/particle/class_2/efficiency": 0.060639072209596634, "metrics/particle/class_2/pid_accuracy": 0.005988024175167084, "metrics/particle/class_3/efficiency": 0.3370234966278076, "metrics/particle/class_3/pid_accuracy": 0.9518004059791565, "metrics/particle/class_4/efficiency": 0.7993197441101074, "metrics/particle/class_4/pid_accuracy": 0.059574469923973083, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.1315789520740509, "metrics/particle/count/bias_mean": -23.84895896911621, "metrics/particle/count/mae": 23.88541603088379, "metrics/particle/count/prediction_mean": 55.71744918823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31896427273750305, "metrics/particle/event/energy_response_mean": 0.6848546862602234, "metrics/particle/event/met_abs_error": 15.351801872253418, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2713244557380676, "metrics/particle/event/scalar_pt_response_mean": 0.7338408827781677, "metrics/particle/event/vector_pt_closure": 0.11585285514593124, "metrics/particle/matched/delta_eta_abs_mean": 0.00880847591906786, "metrics/particle/matched/delta_phi_abs_mean": 0.004150935914367437, "metrics/particle/matched/delta_r_mean": 0.010675733909010887, "metrics/particle/matched/energy_relative_abs_mean": 0.08619587123394012, "metrics/particle/matched/pid_accuracy": 0.9651899337768555, "metrics/particle/matched/pt_relative_abs_mean": 0.0778675302863121, "metrics/particle/matching/duplicate_fraction": 0.024935150519013405, "metrics/particle/matching/efficiency": 0.60691899061203, "metrics/particle/matching/f1": 0.7139117121696472, "metrics/particle/matching/purity": 0.8667009472846985, "metrics/particle/matching/target_coverage_dr0p05": 0.6090300679206848, "metrics/particle/matching/target_coverage_dr0p10": 0.6325461864471436, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5408218502998352, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6261802911758423}, "step_train_time": 1.4991002082824707, "step_valid_time": 9.646333511999728, "step_total_time": 1907.6362941265106, "elapsed_training_loop_seconds": 1907.6362941265106, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7990834812851746, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28699004102578585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 461, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7463976945244957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8854700854700854, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8100078186082877, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11452991452991453, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6642651296829971, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.788034188034188, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7208756841282252, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7463976945244957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8471880932962077, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26763684290409684, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1457, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1320, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1209, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8582574772431729, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.905971173644475, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8814691151919867, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09402882635552505, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7860858257477243, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8297872340425532, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8073455759599332, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8582574772431729, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8165422687546687, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28746893019475195, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1075, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 950, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 831, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8246527777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8837209302325582, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8531656937584194, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11627906976744186, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7213541666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7730232558139535, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7462954647507858, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8246527777777778, "step_test_time": 20.846187737000037, "step_plot_time": 12.651317387999825, "step_validation_cycle_time": 43.59193608900023} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_70000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..43cf44378d6a6ff0bb5d2fc7b90db3a3daa7b868 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 46.666666666666664, "train": {"Classification_binary": 0.2326442152261734, "Classification": 0.0003038116847164929, "Regression_pt": 0.2377370446920395, "Regression_eta": 0.01815468817949295, "Regression_sin_phi": 0.00431452039629221, "Regression_cos_phi": 0.005493418779224157, "Regression_energy": 0.24134236574172974, "Total": 1.4311224222183228, "Total_unweighted": 0.7399900646996684}, "train_group": {}, "valid": {"Classification_binary": 0.2388629913330078, "Classification": 0.00035139227596422035, "Regression_pt": 0.2559700608253479, "Regression_eta": 0.046886314948399864, "Regression_sin_phi": 0.01059099038441976, "Regression_cos_phi": 0.011129007985194525, "Regression_energy": 0.269279420375824, "Total": 2.576909383138021, "Total_unweighted": 0.8330701781281582, "metrics/particle/class_1/efficiency": 0.9101677536964417, "metrics/particle/class_1/pid_accuracy": 0.9881311655044556, "metrics/particle/class_2/efficiency": 0.06209150329232216, "metrics/particle/class_2/pid_accuracy": 0.05263157933950424, "metrics/particle/class_3/efficiency": 0.42962032556533813, "metrics/particle/class_3/pid_accuracy": 0.9711325764656067, "metrics/particle/class_4/efficiency": 0.9149659872055054, "metrics/particle/class_4/pid_accuracy": 0.29739776253700256, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.38596490025520325, "metrics/particle/count/bias_mean": -20.453125, "metrics/particle/count/mae": 20.47135353088379, "metrics/particle/count/prediction_mean": 59.11328125, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32224586606025696, "metrics/particle/event/energy_response_mean": 0.6799209713935852, "metrics/particle/event/met_abs_error": 14.28707218170166, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2684943377971649, "metrics/particle/event/scalar_pt_response_mean": 0.7341201901435852, "metrics/particle/event/vector_pt_closure": 0.11320805549621582, "metrics/particle/matched/delta_eta_abs_mean": 0.007257523946464062, "metrics/particle/matched/delta_phi_abs_mean": 0.003908228594809771, "metrics/particle/matched/delta_r_mean": 0.009167649783194065, "metrics/particle/matched/energy_relative_abs_mean": 0.07484430074691772, "metrics/particle/matched/pid_accuracy": 0.9732182621955872, "metrics/particle/matched/pt_relative_abs_mean": 0.06358760595321655, "metrics/particle/matching/duplicate_fraction": 0.021850701421499252, "metrics/particle/matching/efficiency": 0.6654229760169983, "metrics/particle/matching/f1": 0.7635626196861267, "metrics/particle/matching/purity": 0.8956584930419922, "metrics/particle/matching/target_coverage_dr0p05": 0.6700705289840698, "metrics/particle/matching/target_coverage_dr0p10": 0.6866971254348755, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5930744409561157, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6787929534912109}, "step_train_time": 1.4977056980133057, "step_valid_time": 9.705950505998771, "step_total_time": 19429.3452937603, "elapsed_training_loop_seconds": 19429.3452937603, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8071080090750432, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28131819733424435, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 590, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 519, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 467, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7478386167146974, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8796610169491526, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8084112149532711, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12033898305084746, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6729106628242075, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7915254237288135, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7274143302180686, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7478386167146974, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.846148667391905, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2545797423164028, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1452, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1330, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1228, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8647594278283485, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9159779614325069, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8896321070234113, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08402203856749312, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7984395318595578, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8457300275482094, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8214046822742475, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8647594278283485, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8064761495855958, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2779424715173956, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1072, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 965, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 875, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8376736111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9001865671641791, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8678057553956835, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09981343283582089, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7595486111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.816231343283582, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7868705035971223, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8376736111111112, "step_test_time": 22.119248707000224, "step_plot_time": 13.23880542400002, "step_validation_cycle_time": 45.53119884300031} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_71000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_71000.json new file mode 100644 index 0000000000000000000000000000000000000000..a90bc14e5692814c1e61b0a5c3c3a1bc68791fc4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_71000.json @@ -0,0 +1 @@ +{"epoch": 47.333333333333336, "train": {"Classification_binary": 0.23031634092330933, "Classification": 0.0003635102475527674, "Regression_pt": 0.28456175327301025, "Regression_eta": 0.009965118020772934, "Regression_sin_phi": 0.004003907088190317, "Regression_cos_phi": 0.005033212248235941, "Regression_energy": 0.29039233922958374, "Total": 1.3492846488952637, "Total_unweighted": 0.8246361810306553}, "train_group": {}, "valid": {"Classification_binary": 0.23872357606887817, "Classification": 0.00034594562991211814, "Regression_pt": 0.2521499991416931, "Regression_eta": 0.046044970552126564, "Regression_sin_phi": 0.010850573579470316, "Regression_cos_phi": 0.011252141247193018, "Regression_energy": 0.26630963881810504, "Total": 2.5838705698649087, "Total_unweighted": 0.8256768450373781, "metrics/particle/class_1/efficiency": 0.9103572368621826, "metrics/particle/class_1/pid_accuracy": 0.9874396920204163, "metrics/particle/class_2/efficiency": 0.06427015364170074, "metrics/particle/class_2/pid_accuracy": 0.06214689090847969, "metrics/particle/class_3/efficiency": 0.4265007972717285, "metrics/particle/class_3/pid_accuracy": 0.9700294137001038, "metrics/particle/class_4/efficiency": 0.9081632494926453, "metrics/particle/class_4/pid_accuracy": 0.32209739089012146, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.5701754093170166, "metrics/particle/count/bias_mean": -20.26692771911621, "metrics/particle/count/mae": 20.27994728088379, "metrics/particle/count/prediction_mean": 59.29948043823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3136427402496338, "metrics/particle/event/energy_response_mean": 0.6890654563903809, "metrics/particle/event/met_abs_error": 14.26363468170166, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25898730754852295, "metrics/particle/event/scalar_pt_response_mean": 0.743495523929596, "metrics/particle/event/vector_pt_closure": 0.11216387897729874, "metrics/particle/matched/delta_eta_abs_mean": 0.007331256754696369, "metrics/particle/matched/delta_phi_abs_mean": 0.0036125020124018192, "metrics/particle/matched/delta_r_mean": 0.009068982675671577, "metrics/particle/matched/energy_relative_abs_mean": 0.07405629754066467, "metrics/particle/matched/pid_accuracy": 0.9730475544929504, "metrics/particle/matched/pt_relative_abs_mean": 0.06434255093336105, "metrics/particle/matching/duplicate_fraction": 0.023780247196555138, "metrics/particle/matching/efficiency": 0.6642447113990784, "metrics/particle/matching/f1": 0.7611885666847229, "metrics/particle/matching/purity": 0.8912652134895325, "metrics/particle/matching/target_coverage_dr0p05": 0.674799919128418, "metrics/particle/matching/target_coverage_dr0p10": 0.6903300881385803, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5898014903068542, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6777946949005127}, "step_train_time": 1.5127301216125488, "step_valid_time": 9.750168111997482, "step_total_time": 19708.160580158234, "elapsed_training_loop_seconds": 19708.160580158234, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8023784271285094, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2756401008906042, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 594, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 525, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 474, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7564841498559077, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8838383838383839, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8152173913043478, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11616161616161616, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6829971181556196, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.797979797979798, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7360248447204969, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7564841498559077, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8689014364773097, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24472217605045699, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1457, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1332, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1244, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8660598179453837, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9142072752230611, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.889482470784641, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08579272477693892, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8088426527958388, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.853809196980096, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8307178631051753, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8660598179453837, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8189600323855742, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2794693725551154, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1077, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 971, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 878, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8428819444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9015784586815228, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8712427097353073, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09842154131847725, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7621527777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8152274837511606, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7877972184836249, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8428819444444444, "step_test_time": 21.753984955001215, "step_plot_time": 13.213587893002114, "step_validation_cycle_time": 45.21007930799897} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_72000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_72000.json new file mode 100644 index 0000000000000000000000000000000000000000..bdc2e96fa1b9ee11d3306c96376d0ec27adc5575 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_72000.json @@ -0,0 +1 @@ +{"epoch": 48.0, "train": {"Classification_binary": 0.23823150992393494, "Classification": 0.00030095758847892284, "Regression_pt": 0.23104169964790344, "Regression_eta": 0.03367801010608673, "Regression_sin_phi": 0.009224573150277138, "Regression_cos_phi": 0.011787704192101955, "Regression_energy": 0.2363162338733673, "Total": 2.3092033863067627, "Total_unweighted": 0.7605806884821504}, "train_group": {}, "valid": {"Classification_binary": 0.2391528288523356, "Classification": 0.0003468771077071627, "Regression_pt": 0.25937549273173016, "Regression_eta": 0.04504819711049398, "Regression_sin_phi": 0.010774720460176468, "Regression_cos_phi": 0.01087280735373497, "Regression_energy": 0.2738637328147888, "Total": 2.5533885955810547, "Total_unweighted": 0.8394346564309672, "metrics/particle/class_1/efficiency": 0.9150004982948303, "metrics/particle/class_1/pid_accuracy": 0.9849143624305725, "metrics/particle/class_2/efficiency": 0.0708061009645462, "metrics/particle/class_2/pid_accuracy": 0.0615384615957737, "metrics/particle/class_3/efficiency": 0.46070152521133423, "metrics/particle/class_3/pid_accuracy": 0.9741535782814026, "metrics/particle/class_4/efficiency": 0.8911564350128174, "metrics/particle/class_4/pid_accuracy": 0.38167938590049744, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.5789473652839661, "metrics/particle/count/bias_mean": -18.00520896911621, "metrics/particle/count/mae": 18.05989646911621, "metrics/particle/count/prediction_mean": 61.56119918823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.2986457943916321, "metrics/particle/event/energy_response_mean": 0.7058281302452087, "metrics/particle/event/met_abs_error": 14.39367961883545, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2448638677597046, "metrics/particle/event/scalar_pt_response_mean": 0.7593176960945129, "metrics/particle/event/vector_pt_closure": 0.1130281388759613, "metrics/particle/matched/delta_eta_abs_mean": 0.00989625509828329, "metrics/particle/matched/delta_phi_abs_mean": 0.003918960690498352, "metrics/particle/matched/delta_r_mean": 0.011639208532869816, "metrics/particle/matched/energy_relative_abs_mean": 0.07676663994789124, "metrics/particle/matched/pid_accuracy": 0.9725563526153564, "metrics/particle/matched/pt_relative_abs_mean": 0.06484236568212509, "metrics/particle/matching/duplicate_fraction": 0.02637534588575363, "metrics/particle/matching/efficiency": 0.6815749406814575, "metrics/particle/matching/f1": 0.7685309648513794, "metrics/particle/matching/purity": 0.8809196352958679, "metrics/particle/matching/target_coverage_dr0p05": 0.6910828351974487, "metrics/particle/matching/target_coverage_dr0p10": 0.7075784802436829, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.60569167137146, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.696319580078125}, "step_train_time": 1.4780056476593018, "step_valid_time": 9.709991027997603, "step_total_time": 19986.67715716362, "elapsed_training_loop_seconds": 19986.67715716362, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8210222389502341, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25800728289804664, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 593, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 474, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7521613832853026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8802698145025295, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8111888111888111, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11973018549747048, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6829971181556196, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7993254637436762, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7365967365967366, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7521613832853026, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.884412898799019, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2399086283590437, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1472, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1343, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1265, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8732119635890767, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9123641304347826, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.892358803986711, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08763586956521739, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8224967490247074, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.859375, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.840531561461794, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8732119635890767, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8319626193302266, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27316203381180093, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1084, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 971, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 892, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8428819444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8957564575645757, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.868515205724508, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10424354243542436, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7743055555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8228782287822878, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7978533094812165, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8428819444444444, "step_test_time": 22.140035846998217, "step_plot_time": 13.25784451099753, "step_validation_cycle_time": 45.58256871900085} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_73000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_73000.json new file mode 100644 index 0000000000000000000000000000000000000000..0223d186750bbb1bf840e97fa77e17efd63a6c18 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_73000.json @@ -0,0 +1 @@ +{"epoch": 48.666666666666664, "train": {"Classification_binary": 0.22728979587554932, "Classification": 0.0004174533241894096, "Regression_pt": 0.22380010783672333, "Regression_eta": 0.03791666030883789, "Regression_sin_phi": 0.008642236702144146, "Regression_cos_phi": 0.008676741272211075, "Regression_energy": 0.23766373097896576, "Total": 2.1618192195892334, "Total_unweighted": 0.7444067262986209}, "train_group": {}, "valid": {"Classification_binary": 0.23871986071268717, "Classification": 0.00034860207233577967, "Regression_pt": 0.26135679086049396, "Regression_eta": 0.047841216127077736, "Regression_sin_phi": 0.010612498968839645, "Regression_cos_phi": 0.010871375600496927, "Regression_energy": 0.2747915784517924, "Total": 2.5811568895975747, "Total_unweighted": 0.8445419227937236, "metrics/particle/class_1/efficiency": 0.9078618884086609, "metrics/particle/class_1/pid_accuracy": 0.9885185360908508, "metrics/particle/class_2/efficiency": 0.061002179980278015, "metrics/particle/class_2/pid_accuracy": 0.011904762126505375, "metrics/particle/class_3/efficiency": 0.4414897561073303, "metrics/particle/class_3/pid_accuracy": 0.9731150269508362, "metrics/particle/class_4/efficiency": 0.9081632494926453, "metrics/particle/class_4/pid_accuracy": 0.31086140871047974, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.35087719559669495, "metrics/particle/count/bias_mean": -19.9453125, "metrics/particle/count/mae": 19.95572853088379, "metrics/particle/count/prediction_mean": 59.62109375, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31116345524787903, "metrics/particle/event/energy_response_mean": 0.6920337080955505, "metrics/particle/event/met_abs_error": 14.67318058013916, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25737807154655457, "metrics/particle/event/scalar_pt_response_mean": 0.7459942698478699, "metrics/particle/event/vector_pt_closure": 0.11337795108556747, "metrics/particle/matched/delta_eta_abs_mean": 0.007819819264113903, "metrics/particle/matched/delta_phi_abs_mean": 0.004641852807253599, "metrics/particle/matched/delta_r_mean": 0.010131954215466976, "metrics/particle/matched/energy_relative_abs_mean": 0.07800312340259552, "metrics/particle/matched/pid_accuracy": 0.9739338755607605, "metrics/particle/matched/pt_relative_abs_mean": 0.06394317001104355, "metrics/particle/matching/duplicate_fraction": 0.023914040997624397, "metrics/particle/matching/efficiency": 0.6692522764205933, "metrics/particle/matching/f1": 0.7651548981666565, "metrics/particle/matching/purity": 0.8931402564048767, "metrics/particle/matching/target_coverage_dr0p05": 0.6767800450325012, "metrics/particle/matching/target_coverage_dr0p10": 0.6925229430198669, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5970510840415955, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6830804944038391}, "step_train_time": 1.4312443733215332, "step_valid_time": 9.726530610998452, "step_total_time": 20264.932259082794, "elapsed_training_loop_seconds": 20264.932259082794, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8095254731886043, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26387985825515037, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 590, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 523, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 473, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7536023054755043, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8864406779661017, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8146417445482866, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1135593220338983, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6815561959654178, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8016949152542373, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7367601246105919, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7536023054755043, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8649890425260878, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24874339968265113, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1468, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1340, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1246, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.871261378413524, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9128065395095368, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8915502328675982, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08719346049046321, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8101430429128739, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8487738419618529, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8290086493679308, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.871261378413524, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.817847644575452, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27731451326202516, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1083, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 969, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 871, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8411458333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8947368421052632, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8671140939597315, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10526315789473684, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7560763888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.804247460757156, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7794183445190157, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8411458333333334, "step_test_time": 22.068843932000163, "step_plot_time": 13.405627043001004, "step_validation_cycle_time": 45.732258988002286} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_74000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_74000.json new file mode 100644 index 0000000000000000000000000000000000000000..04687bd4833afc91ac4f87b7fa9a750494bccba9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_74000.json @@ -0,0 +1 @@ +{"epoch": 49.333333333333336, "train": {"Classification_binary": 0.22626182436943054, "Classification": 0.0002829574514180422, "Regression_pt": 0.20264214277267456, "Regression_eta": 0.018529213964939117, "Regression_sin_phi": 0.005845165811479092, "Regression_cos_phi": 0.009166480973362923, "Regression_energy": 0.21213817596435547, "Total": 1.6975737810134888, "Total_unweighted": 0.6748659613076597}, "train_group": {}, "valid": {"Classification_binary": 0.2393751541773478, "Classification": 0.00035396214419354993, "Regression_pt": 0.2583352327346802, "Regression_eta": 0.04411429166793823, "Regression_sin_phi": 0.010904600222905477, "Regression_cos_phi": 0.010906059294939041, "Regression_energy": 0.2703482111295064, "Total": 2.550785541534424, "Total_unweighted": 0.8343375113715108, "metrics/particle/class_1/efficiency": 0.9080830216407776, "metrics/particle/class_1/pid_accuracy": 0.9888691902160645, "metrics/particle/class_2/efficiency": 0.06681191176176071, "metrics/particle/class_2/pid_accuracy": 0.09239130467176437, "metrics/particle/class_3/efficiency": 0.43764740228652954, "metrics/particle/class_3/pid_accuracy": 0.9691411852836609, "metrics/particle/class_4/efficiency": 0.918367326259613, "metrics/particle/class_4/pid_accuracy": 0.3777777850627899, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.3070175349712372, "metrics/particle/count/bias_mean": -19.65234375, "metrics/particle/count/mae": 19.66275978088379, "metrics/particle/count/prediction_mean": 59.9140625, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.30519601702690125, "metrics/particle/event/energy_response_mean": 0.7009298205375671, "metrics/particle/event/met_abs_error": 14.529006004333496, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25575852394104004, "metrics/particle/event/scalar_pt_response_mean": 0.7490645051002502, "metrics/particle/event/vector_pt_closure": 0.1132097914814949, "metrics/particle/matched/delta_eta_abs_mean": 0.009994275867938995, "metrics/particle/matched/delta_phi_abs_mean": 0.003641524352133274, "metrics/particle/matched/delta_r_mean": 0.011607609689235687, "metrics/particle/matched/energy_relative_abs_mean": 0.07449162006378174, "metrics/particle/matched/pid_accuracy": 0.9733225703239441, "metrics/particle/matched/pt_relative_abs_mean": 0.06382772326469421, "metrics/particle/matching/duplicate_fraction": 0.024275220930576324, "metrics/particle/matching/efficiency": 0.6680249571800232, "metrics/particle/matching/f1": 0.7621474862098694, "metrics/particle/matching/purity": 0.8871430158615112, "metrics/particle/matching/target_coverage_dr0p05": 0.6737198829650879, "metrics/particle/matching/target_coverage_dr0p10": 0.6922774910926819, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5962001085281372, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6823768019676208}, "step_train_time": 1.50728178024292, "step_valid_time": 9.854235893002624, "step_total_time": 20545.649264335632, "elapsed_training_loop_seconds": 20545.649264335632, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.80253035160614, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27044579789451695, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 600, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 524, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 471, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7550432276657061, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8733333333333333, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8098918083462133, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12666666666666668, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6786743515850144, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.785, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7279752704791345, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7550432276657061, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8624867854793231, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26556032303223376, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1473, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1344, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1252, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8738621586475943, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9124236252545825, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8927266688807705, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08757637474541752, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8140442132639792, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8499660556687033, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.831617402856194, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8738621586475943, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8169879070284535, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27592899386886305, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 969, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 869, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8411458333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.898886827458256, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8690582959641255, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10111317254174397, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7543402777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8061224489795918, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7793721973094171, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8411458333333334, "step_test_time": 21.86153970400119, "step_plot_time": 13.248337013999844, "step_validation_cycle_time": 45.448892469001294} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_75000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_75000.json new file mode 100644 index 0000000000000000000000000000000000000000..f4a72c749e76955954fc14f60a6600ffe713c722 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_75000.json @@ -0,0 +1 @@ +{"epoch": 50.0, "train": {"Classification_binary": 0.22830456495285034, "Classification": 0.0002595651603769511, "Regression_pt": 0.24913479387760162, "Regression_eta": 0.014714519493281841, "Regression_sin_phi": 0.004629664123058319, "Regression_cos_phi": 0.008360750041902065, "Regression_energy": 0.25687190890312195, "Total": 1.5745424032211304, "Total_unweighted": 0.7622757665521931}, "train_group": {}, "valid": {"Classification_binary": 0.23807303110758463, "Classification": 0.00034083651068309945, "Regression_pt": 0.25442735354105633, "Regression_eta": 0.04565072556336721, "Regression_sin_phi": 0.010961475471655527, "Regression_cos_phi": 0.011304188519716263, "Regression_energy": 0.2670808434486389, "Total": 2.5885780652364097, "Total_unweighted": 0.827838454162702, "metrics/particle/class_1/efficiency": 0.9067564010620117, "metrics/particle/class_1/pid_accuracy": 0.9893057346343994, "metrics/particle/class_2/efficiency": 0.061002179980278015, "metrics/particle/class_2/pid_accuracy": 0.1011904776096344, "metrics/particle/class_3/efficiency": 0.4286312162876129, "metrics/particle/class_3/pid_accuracy": 0.9712434411048889, "metrics/particle/class_4/efficiency": 0.9081632494926453, "metrics/particle/class_4/pid_accuracy": 0.36329588294029236, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.38053098320961, "metrics/particle/count/bias_mean": -20.45963478088379, "metrics/particle/count/mae": 20.47525978088379, "metrics/particle/count/prediction_mean": 59.10676956176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31412121653556824, "metrics/particle/event/energy_response_mean": 0.6878322958946228, "metrics/particle/event/met_abs_error": 14.215472221374512, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26103487610816956, "metrics/particle/event/scalar_pt_response_mean": 0.7406279444694519, "metrics/particle/event/vector_pt_closure": 0.11198970675468445, "metrics/particle/matched/delta_eta_abs_mean": 0.007240315433591604, "metrics/particle/matched/delta_phi_abs_mean": 0.0038896093610674143, "metrics/particle/matched/delta_r_mean": 0.009155935607850552, "metrics/particle/matched/energy_relative_abs_mean": 0.08051633089780807, "metrics/particle/matched/pid_accuracy": 0.9747791290283203, "metrics/particle/matched/pt_relative_abs_mean": 0.06462768465280533, "metrics/particle/matching/duplicate_fraction": 0.022359782829880714, "metrics/particle/matching/efficiency": 0.6631318926811218, "metrics/particle/matching/f1": 0.7609694004058838, "metrics/particle/matching/purity": 0.8926730155944824, "metrics/particle/matching/target_coverage_dr0p05": 0.6707251071929932, "metrics/particle/matching/target_coverage_dr0p10": 0.6862225532531738, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5923216938972473, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6758800148963928}, "step_train_time": 1.4908623695373535, "step_valid_time": 9.759687974998087, "step_total_time": 20824.04763007164, "elapsed_training_loop_seconds": 20824.04763007164, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8001176833653387, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26459231975075337, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 586, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 465, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7463976945244957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8839590443686007, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.809375, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11604095563139932, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.670028818443804, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7935153583617748, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7265625, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7463976945244957, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8632250280438512, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2570747291381835, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1455, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1336, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1249, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8686605981794538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9182130584192439, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8927497494153024, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08178694158075601, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8120936280884266, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8584192439862542, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8346140995656532, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8686605981794538, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8123687536522085, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2837498141049424, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1075, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 972, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 874, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.84375, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9041860465116279, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8729232150875618, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09581395348837209, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7586805555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8130232558139535, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7849124382577458, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.84375, "step_test_time": 22.239744362999772, "step_plot_time": 13.234530596997502, "step_validation_cycle_time": 45.72678796099717} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_76000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_76000.json new file mode 100644 index 0000000000000000000000000000000000000000..e4d3c3a0e86ae60dd91233034dac979edd449532 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_76000.json @@ -0,0 +1 @@ +{"epoch": 50.666666666666664, "train": {"Classification_binary": 0.22319629788398743, "Classification": 0.0002249028329970315, "Regression_pt": 0.24391525983810425, "Regression_eta": 0.03449724242091179, "Regression_sin_phi": 0.009200300090014935, "Regression_cos_phi": 0.00924550648778677, "Regression_energy": 0.2658173441886902, "Total": 2.158789873123169, "Total_unweighted": 0.7860968537424924}, "train_group": {}, "valid": {"Classification_binary": 0.23930921157201132, "Classification": 0.0003531684245293339, "Regression_pt": 0.2545899947484334, "Regression_eta": 0.047616208593050637, "Regression_sin_phi": 0.011004887521266937, "Regression_cos_phi": 0.011011071503162384, "Regression_energy": 0.26772085825602215, "Total": 2.604664166768392, "Total_unweighted": 0.8316054006184761, "metrics/particle/class_1/efficiency": 0.9056192636489868, "metrics/particle/class_1/pid_accuracy": 0.9899898767471313, "metrics/particle/class_2/efficiency": 0.061002179980278015, "metrics/particle/class_2/pid_accuracy": 0.0892857164144516, "metrics/particle/class_3/efficiency": 0.39191964268684387, "metrics/particle/class_3/pid_accuracy": 0.968841016292572, "metrics/particle/class_4/efficiency": 0.9149659872055054, "metrics/particle/class_4/pid_accuracy": 0.33085501194000244, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.5398229956626892, "metrics/particle/count/bias_mean": -22.13411521911621, "metrics/particle/count/mae": 22.13671875, "metrics/particle/count/prediction_mean": 57.43229293823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32247909903526306, "metrics/particle/event/energy_response_mean": 0.6794948577880859, "metrics/particle/event/met_abs_error": 14.353470802307129, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2710765302181244, "metrics/particle/event/scalar_pt_response_mean": 0.7312929630279541, "metrics/particle/event/vector_pt_closure": 0.11264286190271378, "metrics/particle/matched/delta_eta_abs_mean": 0.00782959908246994, "metrics/particle/matched/delta_phi_abs_mean": 0.003939996473491192, "metrics/particle/matched/delta_r_mean": 0.009733302518725395, "metrics/particle/matched/energy_relative_abs_mean": 0.07522828876972198, "metrics/particle/matched/pid_accuracy": 0.9748753905296326, "metrics/particle/matched/pt_relative_abs_mean": 0.06180354952812195, "metrics/particle/matching/duplicate_fraction": 0.023011699318885803, "metrics/particle/matching/efficiency": 0.6467835307121277, "metrics/particle/matching/f1": 0.7512807250022888, "metrics/particle/matching/purity": 0.8960506319999695, "metrics/particle/matching/target_coverage_dr0p05": 0.6554568409919739, "metrics/particle/matching/target_coverage_dr0p10": 0.6709215044975281, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5770042538642883, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6595152616500854}, "step_train_time": 2.5749919414520264, "step_valid_time": 9.71061889399789, "step_total_time": 21104.549730300903, "elapsed_training_loop_seconds": 21104.549730300903, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7909100987009159, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27715176031972644, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 589, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 520, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 459, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7492795389048992, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8828522920203735, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8106001558846454, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11714770797962648, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6613832853025937, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7792869269949066, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7155105222135619, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7492795389048992, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8564190390723716, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25661269320094005, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1454, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1327, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1230, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8628088426527959, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9126547455295736, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8870320855614974, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0873452544704264, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7997399219765929, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8459422283356258, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8221925133689839, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8628088426527959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.800856600600541, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.270679704926013, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1076, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 972, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 861, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.84375, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9033457249070632, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8725314183123878, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09665427509293681, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7473958333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.800185873605948, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7728904847396768, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.84375, "step_test_time": 22.223599859000387, "step_plot_time": 13.558930226001394, "step_validation_cycle_time": 45.99556046200087} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_77000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_77000.json new file mode 100644 index 0000000000000000000000000000000000000000..226389829fe69c5c82eaa9e81a3d37feffdc085f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_77000.json @@ -0,0 +1 @@ +{"epoch": 51.333333333333336, "train": {"Classification_binary": 0.23281055688858032, "Classification": 0.00040629913564771414, "Regression_pt": 0.2610820233821869, "Regression_eta": 0.012707902118563652, "Regression_sin_phi": 0.007040001917630434, "Regression_cos_phi": 0.008926913142204285, "Regression_energy": 0.2548965811729431, "Total": 1.7833731174468994, "Total_unweighted": 0.7778702777577564}, "train_group": {}, "valid": {"Classification_binary": 0.23850021759668985, "Classification": 0.00034077589710553485, "Regression_pt": 0.2513766288757324, "Regression_eta": 0.048338234424591064, "Regression_sin_phi": 0.01089911162853241, "Regression_cos_phi": 0.010808014621337255, "Regression_energy": 0.26514047384262085, "Total": 2.5878448486328125, "Total_unweighted": 0.8254034568866094, "metrics/particle/class_1/efficiency": 0.9075460433959961, "metrics/particle/class_1/pid_accuracy": 0.9870179295539856, "metrics/particle/class_2/efficiency": 0.06354393810033798, "metrics/particle/class_2/pid_accuracy": 0.034285712987184525, "metrics/particle/class_3/efficiency": 0.44822338223457336, "metrics/particle/class_3/pid_accuracy": 0.9737735390663147, "metrics/particle/class_4/efficiency": 0.9149659872055054, "metrics/particle/class_4/pid_accuracy": 0.39405205845832825, "metrics/particle/class_5/efficiency": 0.9824561476707458, "metrics/particle/class_5/pid_accuracy": 0.5357142686843872, "metrics/particle/count/bias_mean": -19.36067771911621, "metrics/particle/count/mae": 19.38150978088379, "metrics/particle/count/prediction_mean": 60.20573043823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3078116476535797, "metrics/particle/event/energy_response_mean": 0.6989105343818665, "metrics/particle/event/met_abs_error": 14.497977256774902, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25331196188926697, "metrics/particle/event/scalar_pt_response_mean": 0.7521288990974426, "metrics/particle/event/vector_pt_closure": 0.11376172304153442, "metrics/particle/matched/delta_eta_abs_mean": 0.007564940489828587, "metrics/particle/matched/delta_phi_abs_mean": 0.003693416016176343, "metrics/particle/matched/delta_r_mean": 0.009298188611865044, "metrics/particle/matched/energy_relative_abs_mean": 0.07531736046075821, "metrics/particle/matched/pid_accuracy": 0.9740443229675293, "metrics/particle/matched/pt_relative_abs_mean": 0.06428290903568268, "metrics/particle/matching/duplicate_fraction": 0.025303862988948822, "metrics/particle/matching/efficiency": 0.672099769115448, "metrics/particle/matching/f1": 0.7651963233947754, "metrics/particle/matching/purity": 0.8882304430007935, "metrics/particle/matching/target_coverage_dr0p05": 0.6813949346542358, "metrics/particle/matching/target_coverage_dr0p10": 0.6972687244415283, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.597427487373352, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6864843368530273}, "step_train_time": 2.423868179321289, "step_valid_time": 9.782735117998527, "step_total_time": 21385.004422187805, "elapsed_training_loop_seconds": 21385.004422187805, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.811587353818843, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2674444958102047, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 598, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 529, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 480, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7622478386167147, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8846153846153846, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8188854489164087, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11538461538461539, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.69164265129683, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.802675585284281, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7430340557275542, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7622478386167147, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8686940313159355, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2497416853777713, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1470, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1352, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1269, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8790637191157347, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9197278911564626, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.898936170212766, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08027210884353742, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8250975292587777, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.863265306122449, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.84375, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8790637191157347, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8119908731315357, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28905268074389767, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 969, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 875, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8411458333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.898886827458256, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8690582959641255, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10111317254174397, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7595486111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8116883116883117, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7847533632286996, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8411458333333334, "step_test_time": 22.138483793998603, "step_plot_time": 13.526236096000503, "step_validation_cycle_time": 45.93008312500024} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_78000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_78000.json new file mode 100644 index 0000000000000000000000000000000000000000..55947e1c6d725858035dbacf7e62a61a9df185d9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_78000.json @@ -0,0 +1 @@ +{"epoch": 52.0, "train": {"Classification_binary": 0.22383147478103638, "Classification": 0.000325460743624717, "Regression_pt": 0.29190966486930847, "Regression_eta": 0.026755712926387787, "Regression_sin_phi": 0.004619615152478218, "Regression_cos_phi": 0.005748351104557514, "Regression_energy": 0.32131972908973694, "Total": 1.654312014579773, "Total_unweighted": 0.87451000866713}, "train_group": {}, "valid": {"Classification_binary": 0.23930917183558145, "Classification": 0.0003469859172279636, "Regression_pt": 0.25713109970092773, "Regression_eta": 0.04667510588963827, "Regression_sin_phi": 0.011440105736255646, "Regression_cos_phi": 0.011576638867457708, "Regression_energy": 0.2710304260253906, "Total": 2.654871940612793, "Total_unweighted": 0.8375095339724794, "metrics/particle/class_1/efficiency": 0.9161691665649414, "metrics/particle/class_1/pid_accuracy": 0.9875538945198059, "metrics/particle/class_2/efficiency": 0.07443717867136002, "metrics/particle/class_2/pid_accuracy": 0.1317073106765747, "metrics/particle/class_3/efficiency": 0.4387887120246887, "metrics/particle/class_3/pid_accuracy": 0.9713889360427856, "metrics/particle/class_4/efficiency": 0.9115646481513977, "metrics/particle/class_4/pid_accuracy": 0.33582088351249695, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.38053098320961, "metrics/particle/count/bias_mean": -18.5859375, "metrics/particle/count/mae": 18.61197853088379, "metrics/particle/count/prediction_mean": 60.98046875, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3001573383808136, "metrics/particle/event/energy_response_mean": 0.7042225003242493, "metrics/particle/event/met_abs_error": 14.090302467346191, "metrics/particle/event/scalar_pt_relative_abs_error": 0.24397259950637817, "metrics/particle/event/scalar_pt_response_mean": 0.7593966126441956, "metrics/particle/event/vector_pt_closure": 0.11040285229682922, "metrics/particle/matched/delta_eta_abs_mean": 0.007554427720606327, "metrics/particle/matched/delta_phi_abs_mean": 0.004114875569939613, "metrics/particle/matched/delta_r_mean": 0.009553186595439911, "metrics/particle/matched/energy_relative_abs_mean": 0.07435550540685654, "metrics/particle/matched/pid_accuracy": 0.9728388786315918, "metrics/particle/matched/pt_relative_abs_mean": 0.06446287035942078, "metrics/particle/matching/duplicate_fraction": 0.025473490357398987, "metrics/particle/matching/efficiency": 0.6729997992515564, "metrics/particle/matching/f1": 0.761997401714325, "metrics/particle/matching/purity": 0.8781201243400574, "metrics/particle/matching/target_coverage_dr0p05": 0.6853224635124207, "metrics/particle/matching/target_coverage_dr0p10": 0.7016381025314331, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5977711081504822, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6873025894165039}, "step_train_time": 2.890089511871338, "step_valid_time": 9.745234144000278, "step_total_time": 21666.29670572281, "elapsed_training_loop_seconds": 21666.29670572281, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8215644818581388, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2698584284942316, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 598, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 537, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 489, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7737752161383286, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8979933110367893, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8312693498452013, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1020066889632107, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7046109510086456, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8177257525083612, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7569659442724458, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7737752161383286, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8701076416412244, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24967472739492613, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1471, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1352, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1273, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8790637191157347, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9191026512576479, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.898637421070123, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08089734874235215, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8276983094928478, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8653976886471788, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.846128281821203, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8790637191157347, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8262004613995506, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.277310334291338, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1089, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 978, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 886, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8489583333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8980716253443526, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8728246318607764, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10192837465564739, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7690972222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8135904499540864, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7907184292726461, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8489583333333334, "step_test_time": 23.150657728001534, "step_plot_time": 13.638480157998856, "step_validation_cycle_time": 47.01188263900258} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_79000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_79000.json new file mode 100644 index 0000000000000000000000000000000000000000..ef7e2f553dd0d35f7665f98f4dcc8d0ce2efd77a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_79000.json @@ -0,0 +1 @@ +{"epoch": 52.666666666666664, "train": {"Classification_binary": 0.2634388506412506, "Classification": 0.00035416384343989193, "Regression_pt": 0.3092687427997589, "Regression_eta": 0.012379136867821217, "Regression_sin_phi": 0.006371645722538233, "Regression_cos_phi": 0.0072191022336483, "Regression_energy": 0.3109031915664673, "Total": 1.715645432472229, "Total_unweighted": 0.9099348336749244}, "train_group": {}, "valid": {"Classification_binary": 0.23882742722829184, "Classification": 0.00034351192880421877, "Regression_pt": 0.2598371108373006, "Regression_eta": 0.04968013366063436, "Regression_sin_phi": 0.011125380794207254, "Regression_cos_phi": 0.011135070274273554, "Regression_energy": 0.2732524474461873, "Total": 2.6497036616007485, "Total_unweighted": 0.844201082169699, "metrics/particle/class_1/efficiency": 0.9085568189620972, "metrics/particle/class_1/pid_accuracy": 0.9887359142303467, "metrics/particle/class_2/efficiency": 0.06572258472442627, "metrics/particle/class_2/pid_accuracy": 0.07734806835651398, "metrics/particle/class_3/efficiency": 0.4224301874637604, "metrics/particle/class_3/pid_accuracy": 0.971721887588501, "metrics/particle/class_4/efficiency": 0.9047619104385376, "metrics/particle/class_4/pid_accuracy": 0.3947368562221527, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.2719298303127289, "metrics/particle/count/bias_mean": -20.81380271911621, "metrics/particle/count/mae": 20.82682228088379, "metrics/particle/count/prediction_mean": 58.75260543823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3093598783016205, "metrics/particle/event/energy_response_mean": 0.6951743960380554, "metrics/particle/event/met_abs_error": 14.091217994689941, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2557632029056549, "metrics/particle/event/scalar_pt_response_mean": 0.7477293610572815, "metrics/particle/event/vector_pt_closure": 0.11192917078733444, "metrics/particle/matched/delta_eta_abs_mean": 0.0076003121212124825, "metrics/particle/matched/delta_phi_abs_mean": 0.004300359636545181, "metrics/particle/matched/delta_r_mean": 0.009733662940561771, "metrics/particle/matched/energy_relative_abs_mean": 0.0727560743689537, "metrics/particle/matched/pid_accuracy": 0.9740532636642456, "metrics/particle/matched/pt_relative_abs_mean": 0.06185688078403473, "metrics/particle/matching/duplicate_fraction": 0.024599973112344742, "metrics/particle/matching/efficiency": 0.661609947681427, "metrics/particle/matching/f1": 0.7611669301986694, "metrics/particle/matching/purity": 0.8959931135177612, "metrics/particle/matching/target_coverage_dr0p05": 0.6693668365478516, "metrics/particle/matching/target_coverage_dr0p10": 0.6853061318397522, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5916670560836792, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6747017502784729}, "step_train_time": 2.7704224586486816, "step_valid_time": 9.87027426900022, "step_total_time": 21948.260313987732, "elapsed_training_loop_seconds": 21948.260313987732, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8065813329338727, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27313981498825235, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 592, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 521, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 472, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7507204610951008, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8800675675675675, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8102643856920684, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11993243243243243, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6801152737752162, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7972972972972973, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7340590979782271, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7507204610951008, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8708597340604929, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24556702899969718, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1462, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1342, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1261, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8725617685305592, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9179206566347469, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8946666666666667, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08207934336525308, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8198959687906372, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8625170998632011, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8406666666666667, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8725617685305592, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8176804832131221, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2769202190684582, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1072, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 962, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 870, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8350694444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8973880597014925, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8651079136690647, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10261194029850747, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7552083333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8115671641791045, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7823741007194245, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8350694444444444, "step_test_time": 22.359872281998832, "step_plot_time": 13.58999793899784, "step_validation_cycle_time": 46.33294398699945} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_8000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..389c5a272be6786d9206580126bf4d8e35a5b7a0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 5.333333333333333, "train": {"Classification_binary": 0.2848549783229828, "Classification": 0.0005570476641878486, "Regression_pt": 0.43374449014663696, "Regression_eta": 0.036814022809267044, "Regression_sin_phi": 0.01259958278387785, "Regression_cos_phi": 0.020391564816236496, "Regression_energy": 0.4546658992767334, "Total": 3.4352505207061768, "Total_unweighted": 1.2436275858199224}, "train_group": {}, "valid": {"Classification_binary": 0.2655179301897685, "Classification": 0.0004848526247466604, "Regression_pt": 0.370275338490804, "Regression_eta": 0.055301547050476074, "Regression_sin_phi": 0.013410394390424093, "Regression_cos_phi": 0.013607940326134363, "Regression_energy": 0.38593196868896484, "Total": 3.210012753804525, "Total_unweighted": 1.1045299717613186, "metrics/particle/class_1/efficiency": 0.8799393773078918, "metrics/particle/class_1/pid_accuracy": 0.9859645366668701, "metrics/particle/class_2/efficiency": 0.05809731408953667, "metrics/particle/class_2/pid_accuracy": 0.012500000186264515, "metrics/particle/class_3/efficiency": 0.3350072205066681, "metrics/particle/class_3/pid_accuracy": 0.9495798349380493, "metrics/particle/class_4/efficiency": 0.8095238208770752, "metrics/particle/class_4/pid_accuracy": 0.04621848836541176, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.12280701845884323, "metrics/particle/count/bias_mean": -23.91145896911621, "metrics/particle/count/mae": 23.9453125, "metrics/particle/count/prediction_mean": 55.65494918823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32711926102638245, "metrics/particle/event/energy_response_mean": 0.6752383708953857, "metrics/particle/event/met_abs_error": 15.147181510925293, "metrics/particle/event/scalar_pt_relative_abs_error": 0.27689090371131897, "metrics/particle/event/scalar_pt_response_mean": 0.7271630764007568, "metrics/particle/event/vector_pt_closure": 0.11598017811775208, "metrics/particle/matched/delta_eta_abs_mean": 0.008500060997903347, "metrics/particle/matched/delta_phi_abs_mean": 0.004067478701472282, "metrics/particle/matched/delta_r_mean": 0.010345499962568283, "metrics/particle/matched/energy_relative_abs_mean": 0.08212067931890488, "metrics/particle/matched/pid_accuracy": 0.9644932150840759, "metrics/particle/matched/pt_relative_abs_mean": 0.07513347268104553, "metrics/particle/matching/duplicate_fraction": 0.024027325212955475, "metrics/particle/matching/efficiency": 0.6083754897117615, "metrics/particle/matching/f1": 0.7159557342529297, "metrics/particle/matching/purity": 0.8697564601898193, "metrics/particle/matching/target_coverage_dr0p05": 0.611501157283783, "metrics/particle/matching/target_coverage_dr0p10": 0.6323989033699036, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.544356644153595, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6269330978393555}, "step_train_time": 1.3691422939300537, "step_valid_time": 9.633939333000399, "step_total_time": 2184.359518289566, "elapsed_training_loop_seconds": 2184.359518289566, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7970677000804475, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.280331115422504, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 581, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 511, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 453, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7363112391930836, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8795180722891566, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8015686274509803, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12048192771084337, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6527377521613833, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7796901893287436, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7105882352941176, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7363112391930836, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8368108295280985, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.267820997437637, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1444, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1310, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1194, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8517555266579974, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.907202216066482, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8786049631120053, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09279778393351801, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.776332899869961, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8268698060941828, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8008048289738431, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8517555266579974, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8063141823434953, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2881428584724699, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1069, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 937, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 822, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8133680555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8765201122544434, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8437640702386312, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1234798877455566, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7135416666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7689429373246024, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7402071139126519, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8133680555555556, "step_test_time": 20.805572385999767, "step_plot_time": 12.639604824999878, "step_validation_cycle_time": 43.52176079200035} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_80000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..38cc472fc80407a2ea2f6d7378a956756e1fae2a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 53.333333333333336, "train": {"Classification_binary": 0.23440730571746826, "Classification": 0.0003147282113786787, "Regression_pt": 0.2837843596935272, "Regression_eta": 0.031064877286553383, "Regression_sin_phi": 0.0030303080566227436, "Regression_cos_phi": 0.004676945973187685, "Regression_energy": 0.280644029378891, "Total": 1.525862693786621, "Total_unweighted": 0.837922554317629}, "train_group": {}, "valid": {"Classification_binary": 0.23787583907445273, "Classification": 0.00034655599544445675, "Regression_pt": 0.2565834919611613, "Regression_eta": 0.047802274425824486, "Regression_sin_phi": 0.010445079455773035, "Regression_cos_phi": 0.010813862085342407, "Regression_energy": 0.271255095799764, "Total": 2.5603747367858887, "Total_unweighted": 0.8351221987977624, "metrics/particle/class_1/efficiency": 0.9123471975326538, "metrics/particle/class_1/pid_accuracy": 0.9881595373153687, "metrics/particle/class_2/efficiency": 0.068990558385849, "metrics/particle/class_2/pid_accuracy": 0.12105263024568558, "metrics/particle/class_3/efficiency": 0.4549570083618164, "metrics/particle/class_3/pid_accuracy": 0.9739944934844971, "metrics/particle/class_4/efficiency": 0.9081632494926453, "metrics/particle/class_4/pid_accuracy": 0.3408239781856537, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.429824560880661, "metrics/particle/count/bias_mean": -18.83854103088379, "metrics/particle/count/mae": 18.859375, "metrics/particle/count/prediction_mean": 60.72786331176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31632813811302185, "metrics/particle/event/energy_response_mean": 0.688612699508667, "metrics/particle/event/met_abs_error": 13.8972806930542, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2628593146800995, "metrics/particle/event/scalar_pt_response_mean": 0.741239070892334, "metrics/particle/event/vector_pt_closure": 0.11099433898925781, "metrics/particle/matched/delta_eta_abs_mean": 0.007700663059949875, "metrics/particle/matched/delta_phi_abs_mean": 0.0042208218947052956, "metrics/particle/matched/delta_r_mean": 0.009788185358047485, "metrics/particle/matched/energy_relative_abs_mean": 0.07675346732139587, "metrics/particle/matched/pid_accuracy": 0.9743806719779968, "metrics/particle/matched/pt_relative_abs_mean": 0.06826614588499069, "metrics/particle/matching/duplicate_fraction": 0.026522867381572723, "metrics/particle/matching/efficiency": 0.6777292490005493, "metrics/particle/matching/f1": 0.7687338590621948, "metrics/particle/matching/purity": 0.8879693150520325, "metrics/particle/matching/target_coverage_dr0p05": 0.6836532354354858, "metrics/particle/matching/target_coverage_dr0p10": 0.7008035182952881, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6069353818893433, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6926047801971436}, "step_train_time": 2.338963270187378, "step_valid_time": 9.777434907999123, "step_total_time": 22228.679480552673, "elapsed_training_loop_seconds": 22228.679480552673, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8013791689731495, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26614571166541623, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 598, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 530, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 480, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7636887608069164, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8862876254180602, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8204334365325078, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11371237458193979, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.69164265129683, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.802675585284281, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7430340557275542, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7636887608069164, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.858112619324287, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24051250512830513, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1468, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1342, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1262, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8725617685305592, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9141689373297003, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8928809048569527, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08583106267029973, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8205461638491548, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8596730245231607, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8396540252827678, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8725617685305592, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8118730385092445, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27852242672793026, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 978, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 872, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8489583333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9072356215213359, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8771300448430494, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09276437847866419, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7569444444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8089053803339518, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7820627802690583, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8489583333333334, "step_test_time": 22.1748444190016, "step_plot_time": 13.2841822219998, "step_validation_cycle_time": 45.72286523100047} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_81000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_81000.json new file mode 100644 index 0000000000000000000000000000000000000000..553ffbc58c007736d540221f32108cac937fac4e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_81000.json @@ -0,0 +1 @@ +{"epoch": 54.0, "train": {"Classification_binary": 0.2351849377155304, "Classification": 0.00029691492090933025, "Regression_pt": 0.2775030732154846, "Regression_eta": 0.010784820653498173, "Regression_sin_phi": 0.0070349061861634254, "Regression_cos_phi": 0.008526461198925972, "Regression_energy": 0.26785045862197876, "Total": 1.7250041961669922, "Total_unweighted": 0.8071815725124907}, "train_group": {}, "valid": {"Classification_binary": 0.23721607526143393, "Classification": 0.00034418480936437845, "Regression_pt": 0.2506983280181885, "Regression_eta": 0.04831668237845103, "Regression_sin_phi": 0.010688929508129755, "Regression_cos_phi": 0.010756271580855051, "Regression_energy": 0.2650402784347534, "Total": 2.570474147796631, "Total_unweighted": 0.823060749991176, "metrics/particle/class_1/efficiency": 0.908683180809021, "metrics/particle/class_1/pid_accuracy": 0.9887027144432068, "metrics/particle/class_2/efficiency": 0.0671750158071518, "metrics/particle/class_2/pid_accuracy": 0.09189189225435257, "metrics/particle/class_3/efficiency": 0.45046794414520264, "metrics/particle/class_3/pid_accuracy": 0.974326491355896, "metrics/particle/class_4/efficiency": 0.9081632494926453, "metrics/particle/class_4/pid_accuracy": 0.29588013887405396, "metrics/particle/class_5/efficiency": 0.9824561476707458, "metrics/particle/class_5/pid_accuracy": 0.3660714328289032, "metrics/particle/count/bias_mean": -19.28515625, "metrics/particle/count/mae": 19.30078125, "metrics/particle/count/prediction_mean": 60.28125, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3070540130138397, "metrics/particle/event/energy_response_mean": 0.6989670395851135, "metrics/particle/event/met_abs_error": 14.258770942687988, "metrics/particle/event/scalar_pt_relative_abs_error": 0.24967823922634125, "metrics/particle/event/scalar_pt_response_mean": 0.7546575665473938, "metrics/particle/event/vector_pt_closure": 0.11369507759809494, "metrics/particle/matched/delta_eta_abs_mean": 0.008375938981771469, "metrics/particle/matched/delta_phi_abs_mean": 0.003871126566082239, "metrics/particle/matched/delta_r_mean": 0.01014331728219986, "metrics/particle/matched/energy_relative_abs_mean": 0.07588200271129608, "metrics/particle/matched/pid_accuracy": 0.9743521213531494, "metrics/particle/matched/pt_relative_abs_mean": 0.06401237100362778, "metrics/particle/matching/duplicate_fraction": 0.025898566469550133, "metrics/particle/matching/efficiency": 0.6737853288650513, "metrics/particle/matching/f1": 0.7667011022567749, "metrics/particle/matching/purity": 0.8893424868583679, "metrics/particle/matching/target_coverage_dr0p05": 0.6830313801765442, "metrics/particle/matching/target_coverage_dr0p10": 0.7003289461135864, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5999967455863953, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6876789927482605}, "step_train_time": 3.0539515018463135, "step_valid_time": 9.744454273000883, "step_total_time": 22509.068277597427, "elapsed_training_loop_seconds": 22509.068277597427, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8062680277074471, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2804158573671415, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 592, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 525, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 480, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7564841498559077, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8868243243243243, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8164852255054432, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11317567567567567, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.69164265129683, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8108108108108109, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7465007776049767, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7564841498559077, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8797585052076758, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23563215496174805, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1476, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1348, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1269, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8764629388816645, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9132791327913279, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8944923689449237, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08672086720867209, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8250975292587777, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8597560975609756, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8420703384207033, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8764629388816645, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8277259209574759, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2876136317720519, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1081, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 973, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 878, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8446180555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9000925069380203, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8714733542319749, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09990749306197964, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7621527777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8122109158186864, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7863860277653381, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8446180555555556, "step_test_time": 22.411725330999616, "step_plot_time": 13.318414875000599, "step_validation_cycle_time": 45.96024401499744} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_82000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_82000.json new file mode 100644 index 0000000000000000000000000000000000000000..e2110b0c8f0500e7be618ae60ede458be60cd6c0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_82000.json @@ -0,0 +1 @@ +{"epoch": 54.666666666666664, "train": {"Classification_binary": 0.2614928185939789, "Classification": 0.0003477326245047152, "Regression_pt": 0.23817722499370575, "Regression_eta": 0.04942592233419418, "Regression_sin_phi": 0.012805572710931301, "Regression_cos_phi": 0.010863756760954857, "Regression_energy": 0.24781115353107452, "Total": 2.7317700386047363, "Total_unweighted": 0.8209241815493442}, "train_group": {}, "valid": {"Classification_binary": 0.23745564619700113, "Classification": 0.0003394874899337689, "Regression_pt": 0.2516597906748454, "Regression_eta": 0.04707167545954386, "Regression_sin_phi": 0.010711619009574255, "Regression_cos_phi": 0.010908181468645731, "Regression_energy": 0.26555105050404865, "Total": 2.5645058949788413, "Total_unweighted": 0.8236974508035928, "metrics/particle/class_1/efficiency": 0.9036924839019775, "metrics/particle/class_1/pid_accuracy": 0.9889898896217346, "metrics/particle/class_2/efficiency": 0.060275960713624954, "metrics/particle/class_2/pid_accuracy": 0.0602409653365612, "metrics/particle/class_3/efficiency": 0.4093433618545532, "metrics/particle/class_3/pid_accuracy": 0.9710037112236023, "metrics/particle/class_4/efficiency": 0.918367326259613, "metrics/particle/class_4/pid_accuracy": 0.3444444537162781, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.48672565817832947, "metrics/particle/count/bias_mean": -21.78255271911621, "metrics/particle/count/mae": 21.78255271911621, "metrics/particle/count/prediction_mean": 57.78385543823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3271844685077667, "metrics/particle/event/energy_response_mean": 0.6757509112358093, "metrics/particle/event/met_abs_error": 14.12050724029541, "metrics/particle/event/scalar_pt_relative_abs_error": 0.27697524428367615, "metrics/particle/event/scalar_pt_response_mean": 0.7255859971046448, "metrics/particle/event/vector_pt_closure": 0.11235472559928894, "metrics/particle/matched/delta_eta_abs_mean": 0.007723348680883646, "metrics/particle/matched/delta_phi_abs_mean": 0.0038330303505063057, "metrics/particle/matched/delta_r_mean": 0.009529301896691322, "metrics/particle/matched/energy_relative_abs_mean": 0.07342008501291275, "metrics/particle/matched/pid_accuracy": 0.9744983315467834, "metrics/particle/matched/pt_relative_abs_mean": 0.06416353583335876, "metrics/particle/matching/duplicate_fraction": 0.0201451163738966, "metrics/particle/matching/efficiency": 0.6532639265060425, "metrics/particle/matching/f1": 0.756865918636322, "metrics/particle/matching/purity": 0.8995223045349121, "metrics/particle/matching/target_coverage_dr0p05": 0.6588934063911438, "metrics/particle/matching/target_coverage_dr0p10": 0.674947202205658, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.581619143486023, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6649974584579468}, "step_train_time": 2.731957197189331, "step_valid_time": 9.724413321000611, "step_total_time": 22789.192094564438, "elapsed_training_loop_seconds": 22789.192094564438, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7877482360256277, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.29543003471719154, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 583, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 521, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 460, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7507204610951008, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8936535162950258, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8159749412685983, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10634648370497427, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6628242074927954, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7890222984562607, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7204385277995301, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7507204610951008, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8429618217939308, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2604533311099886, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1451, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1310, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1206, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8517555266579974, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9028256374913852, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8765473402475744, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09717436250861475, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7841352405721717, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8311509303928325, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8069588491134159, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8517555266579974, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7993611124209701, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2698159211352329, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1067, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 953, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 847, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8272569444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8931583880037488, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8589454709328527, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10684161199625117, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7352430555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7938144329896907, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7634069400630915, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8272569444444444, "step_test_time": 21.967221841001447, "step_plot_time": 13.439611205998517, "step_validation_cycle_time": 45.626891759999125} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_83000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_83000.json new file mode 100644 index 0000000000000000000000000000000000000000..e8ad4704141a9b81458a0106fc905849794462e1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_83000.json @@ -0,0 +1 @@ +{"epoch": 55.333333333333336, "train": {"Classification_binary": 0.21957021951675415, "Classification": 0.0002789688587654382, "Regression_pt": 0.2058069407939911, "Regression_eta": 0.02545110695064068, "Regression_sin_phi": 0.004389145877212286, "Regression_cos_phi": 0.0035452900920063257, "Regression_energy": 0.22033938765525818, "Total": 1.3647202253341675, "Total_unweighted": 0.6793810597446281}, "train_group": {}, "valid": {"Classification_binary": 0.23784484465916952, "Classification": 0.0003363622430091103, "Regression_pt": 0.2522195180257161, "Regression_eta": 0.047622943917910256, "Regression_sin_phi": 0.011652945230404535, "Regression_cos_phi": 0.011297474304835001, "Regression_energy": 0.2665507396062215, "Total": 2.654226303100586, "Total_unweighted": 0.827524827987266, "metrics/particle/class_1/efficiency": 0.9063141345977783, "metrics/particle/class_1/pid_accuracy": 0.988254964351654, "metrics/particle/class_2/efficiency": 0.06209150329232216, "metrics/particle/class_2/pid_accuracy": 0.05263157933950424, "metrics/particle/class_3/efficiency": 0.43555504083633423, "metrics/particle/class_3/pid_accuracy": 0.9736221432685852, "metrics/particle/class_4/efficiency": 0.9081632494926453, "metrics/particle/class_4/pid_accuracy": 0.3483146131038666, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.4159291982650757, "metrics/particle/count/bias_mean": -20.70703125, "metrics/particle/count/mae": 20.71224021911621, "metrics/particle/count/prediction_mean": 58.859375, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31921008229255676, "metrics/particle/event/energy_response_mean": 0.6812339425086975, "metrics/particle/event/met_abs_error": 14.158169746398926, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26687172055244446, "metrics/particle/event/scalar_pt_response_mean": 0.7344188094139099, "metrics/particle/event/vector_pt_closure": 0.11033535748720169, "metrics/particle/matched/delta_eta_abs_mean": 0.00822104886174202, "metrics/particle/matched/delta_phi_abs_mean": 0.0038261485751718283, "metrics/particle/matched/delta_r_mean": 0.009973737411201, "metrics/particle/matched/energy_relative_abs_mean": 0.07291831076145172, "metrics/particle/matched/pid_accuracy": 0.9744182229042053, "metrics/particle/matched/pt_relative_abs_mean": 0.06271375715732574, "metrics/particle/matching/duplicate_fraction": 0.021657375618815422, "metrics/particle/matching/efficiency": 0.6659302711486816, "metrics/particle/matching/f1": 0.7655463814735413, "metrics/particle/matching/purity": 0.9002079367637634, "metrics/particle/matching/target_coverage_dr0p05": 0.6710196733474731, "metrics/particle/matching/target_coverage_dr0p10": 0.6870571374893188, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5946781635284424, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6787765622138977}, "step_train_time": 2.5526199340820312, "step_valid_time": 9.706390174000262, "step_total_time": 23069.73931312561, "elapsed_training_loop_seconds": 23069.73931312561, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7940431420668395, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2698219109621345, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 581, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 516, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 471, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7435158501440923, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8881239242685026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8094117647058824, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11187607573149742, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6786743515850144, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.810671256454389, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7388235294117647, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7435158501440923, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.858388078722184, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23359168059546054, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1465, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1328, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1250, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8634590377113134, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9064846416382253, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8844488844488845, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09351535836177474, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.812743823146944, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8532423208191127, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8325008325008325, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8634590377113134, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8137545347523822, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2746156536537284, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1068, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 966, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 873, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8385416666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9044943820224719, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8702702702702703, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09550561797752809, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7578125, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8174157303370787, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7864864864864864, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8385416666666666, "step_test_time": 22.256090168997616, "step_plot_time": 13.296885733001545, "step_validation_cycle_time": 45.76559379099854} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_84000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_84000.json new file mode 100644 index 0000000000000000000000000000000000000000..83ba0c60e34c58843e923009c82bf3acce304b13 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_84000.json @@ -0,0 +1 @@ +{"epoch": 56.0, "train": {"Classification_binary": 0.23400354385375977, "Classification": 0.000328114430885762, "Regression_pt": 0.2906341254711151, "Regression_eta": 0.017262665554881096, "Regression_sin_phi": 0.005715208128094673, "Regression_cos_phi": 0.006822418887168169, "Regression_energy": 0.30048173666000366, "Total": 1.6606532335281372, "Total_unweighted": 0.8552478129859082}, "train_group": {}, "valid": {"Classification_binary": 0.23829887310663858, "Classification": 0.00034694276594867307, "Regression_pt": 0.2578110694885254, "Regression_eta": 0.04926241934299469, "Regression_sin_phi": 0.011795544375975927, "Regression_cos_phi": 0.01156584545969963, "Regression_energy": 0.27148399750391644, "Total": 2.7098633448282876, "Total_unweighted": 0.8405646920436993, "metrics/particle/class_1/efficiency": 0.9131684303283691, "metrics/particle/class_1/pid_accuracy": 0.9894154071807861, "metrics/particle/class_2/efficiency": 0.06245461106300354, "metrics/particle/class_2/pid_accuracy": 0.06395348906517029, "metrics/particle/class_3/efficiency": 0.42794644832611084, "metrics/particle/class_3/pid_accuracy": 0.9704862833023071, "metrics/particle/class_4/efficiency": 0.8979591727256775, "metrics/particle/class_4/pid_accuracy": 0.3219696879386902, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.32743361592292786, "metrics/particle/count/bias_mean": -20.44661521911621, "metrics/particle/count/mae": 20.46224021911621, "metrics/particle/count/prediction_mean": 59.11979293823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3088127672672272, "metrics/particle/event/energy_response_mean": 0.6932690143585205, "metrics/particle/event/met_abs_error": 14.386675834655762, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2535448372364044, "metrics/particle/event/scalar_pt_response_mean": 0.7489995956420898, "metrics/particle/event/vector_pt_closure": 0.1121024563908577, "metrics/particle/matched/delta_eta_abs_mean": 0.0098734050989151, "metrics/particle/matched/delta_phi_abs_mean": 0.0040961517952382565, "metrics/particle/matched/delta_r_mean": 0.011641405522823334, "metrics/particle/matched/energy_relative_abs_mean": 0.07350602746009827, "metrics/particle/matched/pid_accuracy": 0.9741082787513733, "metrics/particle/matched/pt_relative_abs_mean": 0.06136292964220047, "metrics/particle/matching/duplicate_fraction": 0.023720376193523407, "metrics/particle/matching/efficiency": 0.6661757230758667, "metrics/particle/matching/f1": 0.7643905282020569, "metrics/particle/matching/purity": 0.896573007106781, "metrics/particle/matching/target_coverage_dr0p05": 0.6720016002655029, "metrics/particle/matching/target_coverage_dr0p10": 0.6882517337799072, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.597427487373352, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6792020797729492}, "step_train_time": 2.4361255168914795, "step_valid_time": 9.735650820999581, "step_total_time": 23348.971337795258, "elapsed_training_loop_seconds": 23348.971337795258, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8069350670944261, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26429747174791074, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 596, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 519, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 471, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7478386167146974, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8708053691275168, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8046511627906977, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12919463087248323, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6786743515850144, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.790268456375839, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7302325581395349, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7478386167146974, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8588369847791599, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25568685675888614, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1460, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1340, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1246, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.871261378413524, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9178082191780822, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8939292861907938, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0821917808219178, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8101430429128739, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8534246575342466, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8312208138759173, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.871261378413524, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8212211798873474, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27822843561223576, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 967, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 865, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8394097222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8970315398886828, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8672645739910314, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10296846011131726, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7508680555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8024118738404453, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7757847533632287, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8394097222222222, "step_test_time": 22.37967005199971, "step_plot_time": 13.28660837300049, "step_validation_cycle_time": 45.89999335399989} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_85000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_85000.json new file mode 100644 index 0000000000000000000000000000000000000000..5c1f57f07c46e25fd450c7f7aae8ac2e36d47309 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_85000.json @@ -0,0 +1 @@ +{"epoch": 56.666666666666664, "train": {"Classification_binary": 0.2461005002260208, "Classification": 0.00029324277420528233, "Regression_pt": 0.20636394619941711, "Regression_eta": 0.029097190126776695, "Regression_sin_phi": 0.00494087440893054, "Regression_cos_phi": 0.004566475283354521, "Regression_energy": 0.20754565298557281, "Total": 1.5298362970352173, "Total_unweighted": 0.6989078820042778}, "train_group": {}, "valid": {"Classification_binary": 0.23745693763097128, "Classification": 0.00034426955971866846, "Regression_pt": 0.25425124168395996, "Regression_eta": 0.04984015226364136, "Regression_sin_phi": 0.010928839445114136, "Regression_cos_phi": 0.011297933757305145, "Regression_energy": 0.2671802242596944, "Total": 2.640845775604248, "Total_unweighted": 0.831299598600405, "metrics/particle/class_1/efficiency": 0.9090937972068787, "metrics/particle/class_1/pid_accuracy": 0.9874569773674011, "metrics/particle/class_2/efficiency": 0.06681191176176071, "metrics/particle/class_2/pid_accuracy": 0.04891304299235344, "metrics/particle/class_3/efficiency": 0.4336909353733063, "metrics/particle/class_3/pid_accuracy": 0.9680701494216919, "metrics/particle/class_4/efficiency": 0.9013605713844299, "metrics/particle/class_4/pid_accuracy": 0.3396226465702057, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.5, "metrics/particle/count/bias_mean": -19.95833396911621, "metrics/particle/count/mae": 19.9765625, "metrics/particle/count/prediction_mean": 59.60807418823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.2957812249660492, "metrics/particle/event/energy_response_mean": 0.7107685208320618, "metrics/particle/event/met_abs_error": 14.761261940002441, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2452208250761032, "metrics/particle/event/scalar_pt_response_mean": 0.75956130027771, "metrics/particle/event/vector_pt_closure": 0.11408566683530807, "metrics/particle/matched/delta_eta_abs_mean": 0.00780771067366004, "metrics/particle/matched/delta_phi_abs_mean": 0.004024231340736151, "metrics/particle/matched/delta_r_mean": 0.009705349802970886, "metrics/particle/matched/energy_relative_abs_mean": 0.07498568296432495, "metrics/particle/matched/pid_accuracy": 0.972216784954071, "metrics/particle/matched/pt_relative_abs_mean": 0.06483165174722672, "metrics/particle/matching/duplicate_fraction": 0.02411586046218872, "metrics/particle/matching/efficiency": 0.6667648553848267, "metrics/particle/matching/f1": 0.7623823285102844, "metrics/particle/matching/purity": 0.8900150656700134, "metrics/particle/matching/target_coverage_dr0p05": 0.6744071841239929, "metrics/particle/matching/target_coverage_dr0p10": 0.691639244556427, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5942854285240173, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6808058023452759}, "step_train_time": 2.577065944671631, "step_valid_time": 9.717620657000225, "step_total_time": 23629.877794981003, "elapsed_training_loop_seconds": 23629.877794981003, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8270761806978584, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2755728967222316, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 603, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 528, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 476, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.760806916426513, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8756218905472637, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8141865844255975, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12437810945273632, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6858789625360231, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7893864013266998, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7340015420200463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.760806916426513, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8705185781662665, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2578625087502907, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1469, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1344, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1262, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8738621586475943, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9149081007488087, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8939142001995344, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08509189925119129, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8205461638491548, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8590878148400273, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8393747921516461, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8738621586475943, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.834668050072584, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28355441079661925, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1082, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 978, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 881, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8489583333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9038817005545287, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8755595344673232, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09611829944547134, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7647569444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8142329020332717, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7887197851387645, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8489583333333334, "step_test_time": 22.21168081900032, "step_plot_time": 13.671943037999881, "step_validation_cycle_time": 46.10886667399973} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_86000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_86000.json new file mode 100644 index 0000000000000000000000000000000000000000..39ba65de76c6eb8ec1d58fcfcaf0bce3b15e7821 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_86000.json @@ -0,0 +1 @@ +{"epoch": 57.333333333333336, "train": {"Classification_binary": 0.23224122822284698, "Classification": 0.00028100862982682884, "Regression_pt": 0.20800477266311646, "Regression_eta": 0.012487716041505337, "Regression_sin_phi": 0.003639821894466877, "Regression_cos_phi": 0.0038787880912423134, "Regression_energy": 0.2151372879743576, "Total": 1.1801942586898804, "Total_unweighted": 0.6756706235173624}, "train_group": {}, "valid": {"Classification_binary": 0.2377909024556478, "Classification": 0.0003455564146861434, "Regression_pt": 0.259543498357137, "Regression_eta": 0.05025642613569895, "Regression_sin_phi": 0.011028421421845755, "Regression_cos_phi": 0.011206768453121185, "Regression_energy": 0.2723781069119771, "Total": 2.654118855794271, "Total_unweighted": 0.842549680150114, "metrics/particle/class_1/efficiency": 0.9101992845535278, "metrics/particle/class_1/pid_accuracy": 0.9879927635192871, "metrics/particle/class_2/efficiency": 0.05809731408953667, "metrics/particle/class_2/pid_accuracy": 0.05000000074505806, "metrics/particle/class_3/efficiency": 0.4167617857456207, "metrics/particle/class_3/pid_accuracy": 0.9714285731315613, "metrics/particle/class_4/efficiency": 0.9081632494926453, "metrics/particle/class_4/pid_accuracy": 0.36329588294029236, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.5132743120193481, "metrics/particle/count/bias_mean": -21.41666603088379, "metrics/particle/count/mae": 21.42708396911621, "metrics/particle/count/prediction_mean": 58.14973831176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3127521276473999, "metrics/particle/event/energy_response_mean": 0.6915633678436279, "metrics/particle/event/met_abs_error": 14.39424991607666, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25696998834609985, "metrics/particle/event/scalar_pt_response_mean": 0.7480092644691467, "metrics/particle/event/vector_pt_closure": 0.11403431743383408, "metrics/particle/matched/delta_eta_abs_mean": 0.0075142537243664265, "metrics/particle/matched/delta_phi_abs_mean": 0.003592384746298194, "metrics/particle/matched/delta_r_mean": 0.009190035983920097, "metrics/particle/matched/energy_relative_abs_mean": 0.07595334202051163, "metrics/particle/matched/pid_accuracy": 0.9742998480796814, "metrics/particle/matched/pt_relative_abs_mean": 0.06450562924146652, "metrics/particle/matching/duplicate_fraction": 0.02156340330839157, "metrics/particle/matching/efficiency": 0.6596789360046387, "metrics/particle/matching/f1": 0.7622676491737366, "metrics/particle/matching/purity": 0.9026399850845337, "metrics/particle/matching/target_coverage_dr0p05": 0.6691868305206299, "metrics/particle/matching/target_coverage_dr0p10": 0.682982325553894, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5886396169662476, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.670823335647583}, "step_train_time": 2.627739191055298, "step_valid_time": 9.734075378997659, "step_total_time": 23911.04163146019, "elapsed_training_loop_seconds": 23911.04163146019, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8054050676409277, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26922282535498143, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 591, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 469, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7521613832853026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.883248730964467, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8124513618677043, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.116751269035533, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6757925072046109, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7935702199661591, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7299610894941634, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7521613832853026, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8690572154215168, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2522053572966445, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1465, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1342, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1258, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8725617685305592, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9160409556313993, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8937728937728938, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08395904436860069, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8179453836150845, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8587030716723549, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8378288378288379, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8725617685305592, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8195860015023657, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2789250275968941, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1074, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 968, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 875, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8402777777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9013035381750466, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8697214734950584, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09869646182495345, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7595486111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8147113594040968, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7861635220125787, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8402777777777778, "step_test_time": 22.326973375002126, "step_plot_time": 13.50216388500121, "step_validation_cycle_time": 46.072593952001625} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_87000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_87000.json new file mode 100644 index 0000000000000000000000000000000000000000..7c0b23a555a5f8eeecd39b390481574051d65de2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_87000.json @@ -0,0 +1 @@ +{"epoch": 58.0, "train": {"Classification_binary": 0.2101006805896759, "Classification": 0.0003090528189204633, "Regression_pt": 0.1910567283630371, "Regression_eta": 0.025473017245531082, "Regression_sin_phi": 0.004234046209603548, "Regression_cos_phi": 0.003391808131709695, "Regression_energy": 0.199210986495018, "Total": 1.3234068155288696, "Total_unweighted": 0.6337763198534958}, "train_group": {}, "valid": {"Classification_binary": 0.23704904317855835, "Classification": 0.0003451539669185877, "Regression_pt": 0.25783026218414307, "Regression_eta": 0.04947327574094137, "Regression_sin_phi": 0.01091906800866127, "Regression_cos_phi": 0.011194456368684769, "Regression_energy": 0.26968153317769367, "Total": 2.6328543027242026, "Total_unweighted": 0.8364927926256012, "metrics/particle/class_1/efficiency": 0.9150636196136475, "metrics/particle/class_1/pid_accuracy": 0.9879530668258667, "metrics/particle/class_2/efficiency": 0.06753812730312347, "metrics/particle/class_2/pid_accuracy": 0.09677419066429138, "metrics/particle/class_3/efficiency": 0.44620710611343384, "metrics/particle/class_3/pid_accuracy": 0.9703299403190613, "metrics/particle/class_4/efficiency": 0.9149659872055054, "metrics/particle/class_4/pid_accuracy": 0.34944239258766174, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.5526315569877625, "metrics/particle/count/bias_mean": -19.33072853088379, "metrics/particle/count/mae": 19.34895896911621, "metrics/particle/count/prediction_mean": 60.23567581176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3131526708602905, "metrics/particle/event/energy_response_mean": 0.6902129650115967, "metrics/particle/event/met_abs_error": 14.047374725341797, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2594071924686432, "metrics/particle/event/scalar_pt_response_mean": 0.7429230213165283, "metrics/particle/event/vector_pt_closure": 0.11088325828313828, "metrics/particle/matched/delta_eta_abs_mean": 0.0074288384057581425, "metrics/particle/matched/delta_phi_abs_mean": 0.004003757610917091, "metrics/particle/matched/delta_r_mean": 0.009375154040753841, "metrics/particle/matched/energy_relative_abs_mean": 0.07339634746313095, "metrics/particle/matched/pid_accuracy": 0.9735630750656128, "metrics/particle/matched/pt_relative_abs_mean": 0.063498355448246, "metrics/particle/matching/duplicate_fraction": 0.024426622316241264, "metrics/particle/matching/efficiency": 0.6753399968147278, "metrics/particle/matching/f1": 0.7687206864356995, "metrics/particle/matching/purity": 0.8920689225196838, "metrics/particle/matching/target_coverage_dr0p05": 0.6821967959403992, "metrics/particle/matching/target_coverage_dr0p10": 0.6974651217460632, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6032205820083618, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6886935830116272}, "step_train_time": 2.7359347343444824, "step_valid_time": 9.787235766998492, "step_total_time": 24191.638979434967, "elapsed_training_loop_seconds": 24191.638979434967, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8053354215156576, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2621928968309396, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 589, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 524, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 476, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7550432276657061, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8896434634974533, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8168355416991426, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11035653650254669, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6858789625360231, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8081494057724957, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7420109119251753, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7550432276657061, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8539550870805663, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24472382191416886, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1465, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1344, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1250, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8738621586475943, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9174061433447099, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8951048951048951, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08259385665529011, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.812743823146944, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8532423208191127, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8325008325008325, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8738621586475943, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8181896965818354, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27028118886260255, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1075, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 973, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 879, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8446180555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9051162790697674, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8738212842388864, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09488372093023256, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7630208333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8176744186046512, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7894027840143691, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8446180555555556, "step_test_time": 22.509506076003163, "step_plot_time": 13.30429319399991, "step_validation_cycle_time": 46.100861543000065} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_88000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_88000.json new file mode 100644 index 0000000000000000000000000000000000000000..7aad21021305a749f411766694e5203275e9fc62 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_88000.json @@ -0,0 +1 @@ +{"epoch": 58.666666666666664, "train": {"Classification_binary": 0.23874251544475555, "Classification": 0.00025239490787498653, "Regression_pt": 0.222460076212883, "Regression_eta": 0.02004377916455269, "Regression_sin_phi": 0.004187805112451315, "Regression_cos_phi": 0.00333404541015625, "Regression_energy": 0.22796353697776794, "Total": 1.2959306240081787, "Total_unweighted": 0.7169841532304417}, "train_group": {}, "valid": {"Classification_binary": 0.23809059460957846, "Classification": 0.00034445795851449174, "Regression_pt": 0.2570796608924866, "Regression_eta": 0.04581427574157715, "Regression_sin_phi": 0.011335072418053945, "Regression_cos_phi": 0.011291751017173132, "Regression_energy": 0.2696988383928935, "Total": 2.617619355519613, "Total_unweighted": 0.8336546510302774, "metrics/particle/class_1/efficiency": 0.9060614705085754, "metrics/particle/class_1/pid_accuracy": 0.9871361255645752, "metrics/particle/class_2/efficiency": 0.06535948067903519, "metrics/particle/class_2/pid_accuracy": 0.06666667014360428, "metrics/particle/class_3/efficiency": 0.4545004963874817, "metrics/particle/class_3/pid_accuracy": 0.9768979549407959, "metrics/particle/class_4/efficiency": 0.8945578336715698, "metrics/particle/class_4/pid_accuracy": 0.3612167239189148, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.4385964870452881, "metrics/particle/count/bias_mean": -19.8671875, "metrics/particle/count/mae": 19.87760353088379, "metrics/particle/count/prediction_mean": 59.69921875, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3157946765422821, "metrics/particle/event/energy_response_mean": 0.6855735182762146, "metrics/particle/event/met_abs_error": 14.1251220703125, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2603009343147278, "metrics/particle/event/scalar_pt_response_mean": 0.7413709759712219, "metrics/particle/event/vector_pt_closure": 0.11247062683105469, "metrics/particle/matched/delta_eta_abs_mean": 0.007274744100868702, "metrics/particle/matched/delta_phi_abs_mean": 0.0036573633551597595, "metrics/particle/matched/delta_r_mean": 0.009043178521096706, "metrics/particle/matched/energy_relative_abs_mean": 0.07585670799016953, "metrics/particle/matched/pid_accuracy": 0.9746291637420654, "metrics/particle/matched/pt_relative_abs_mean": 0.062040068209171295, "metrics/particle/matching/duplicate_fraction": 0.023228423669934273, "metrics/particle/matching/efficiency": 0.6740471720695496, "metrics/particle/matching/f1": 0.7702045440673828, "metrics/particle/matching/purity": 0.8983620405197144, "metrics/particle/matching/target_coverage_dr0p05": 0.6817713379859924, "metrics/particle/matching/target_coverage_dr0p10": 0.6972196102142334, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6020095944404602, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6871062517166138}, "step_train_time": 2.753751754760742, "step_valid_time": 9.776324858001317, "step_total_time": 24472.770542621613, "elapsed_training_loop_seconds": 24472.770542621613, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8014153397705712, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2728110835934078, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 587, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 524, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 471, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7550432276657061, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8926746166950597, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8181108508977362, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10732538330494037, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6786743515850144, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8023850085178875, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7353629976580797, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7550432276657061, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8768382383382437, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2358606079512926, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1460, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1343, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1263, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8732119635890767, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9198630136986301, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.895930620413609, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08013698630136987, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8211963589076723, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8650684931506849, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8425617078052035, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8732119635890767, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8183482457302202, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28414823732069794, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1073, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 960, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 863, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8333333333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8946877912395154, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8629213483146068, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10531220876048462, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7491319444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.804287045666356, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7757303370786517, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8333333333333334, "step_test_time": 22.119641946999764, "step_plot_time": 13.542475951999222, "step_validation_cycle_time": 45.95485488600025} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_89000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_89000.json new file mode 100644 index 0000000000000000000000000000000000000000..8a925abc9f4b7ffba2197ce4bcf414c847c8029b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_89000.json @@ -0,0 +1 @@ +{"epoch": 59.333333333333336, "train": {"Classification_binary": 0.2398468554019928, "Classification": 0.00028303495491854846, "Regression_pt": 0.23372915387153625, "Regression_eta": 0.009422454982995987, "Regression_sin_phi": 0.003201115410774946, "Regression_cos_phi": 0.004753267392516136, "Regression_energy": 0.23592263460159302, "Total": 1.2001279592514038, "Total_unweighted": 0.7271585166163277}, "train_group": {}, "valid": {"Classification_binary": 0.23806798458099365, "Classification": 0.00034521469691147405, "Regression_pt": 0.25626381238301593, "Regression_eta": 0.04788352052370707, "Regression_sin_phi": 0.0115054522951444, "Regression_cos_phi": 0.01137907306353251, "Regression_energy": 0.26979708671569824, "Total": 2.6600993474324546, "Total_unweighted": 0.8352421442590032, "metrics/particle/class_1/efficiency": 0.9115575551986694, "metrics/particle/class_1/pid_accuracy": 0.9878720641136169, "metrics/particle/class_2/efficiency": 0.0708061009645462, "metrics/particle/class_2/pid_accuracy": 0.06666667014360428, "metrics/particle/class_3/efficiency": 0.4589134752750397, "metrics/particle/class_3/pid_accuracy": 0.9745503067970276, "metrics/particle/class_4/efficiency": 0.9013605713844299, "metrics/particle/class_4/pid_accuracy": 0.3358490467071533, "metrics/particle/class_5/efficiency": 0.9824561476707458, "metrics/particle/class_5/pid_accuracy": 0.4375, "metrics/particle/count/bias_mean": -18.8125, "metrics/particle/count/mae": 18.8359375, "metrics/particle/count/prediction_mean": 60.75390625, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.2933008074760437, "metrics/particle/event/energy_response_mean": 0.7130995392799377, "metrics/particle/event/met_abs_error": 14.566985130310059, "metrics/particle/event/scalar_pt_relative_abs_error": 0.24051357805728912, "metrics/particle/event/scalar_pt_response_mean": 0.7646599411964417, "metrics/particle/event/vector_pt_closure": 0.1139666810631752, "metrics/particle/matched/delta_eta_abs_mean": 0.0083613321185112, "metrics/particle/matched/delta_phi_abs_mean": 0.003712273668497801, "metrics/particle/matched/delta_r_mean": 0.010042465291917324, "metrics/particle/matched/energy_relative_abs_mean": 0.07839005440473557, "metrics/particle/matched/pid_accuracy": 0.9740203619003296, "metrics/particle/matched/pt_relative_abs_mean": 0.0641680657863617, "metrics/particle/matching/duplicate_fraction": 0.026297178119421005, "metrics/particle/matching/efficiency": 0.679038405418396, "metrics/particle/matching/f1": 0.7700759172439575, "metrics/particle/matching/purity": 0.8893032670021057, "metrics/particle/matching/target_coverage_dr0p05": 0.6882517337799072, "metrics/particle/matching/target_coverage_dr0p10": 0.7044528722763062, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6030732989311218, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6931120753288269}, "step_train_time": 2.5665297508239746, "step_valid_time": 9.769892768999853, "step_total_time": 24753.354878664017, "elapsed_training_loop_seconds": 24753.354878664017, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8304731419622353, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28888289794933797, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 595, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 528, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 486, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.760806916426513, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8873949579831932, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8192397207137315, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11260504201680673, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7002881844380403, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8168067226890756, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7540729247478666, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.760806916426513, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8860594811614526, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24483078601038522, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1471, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1351, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1275, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8784135240572172, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9184228416043508, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8979727484214025, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08157715839564922, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8289986996098829, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8667573079537729, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.847457627118644, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8784135240572172, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.848563399260978, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2852095776263256, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1096, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 991, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 900, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8602430555555556, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9041970802919708, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8816725978647687, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09580291970802919, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.78125, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8211678832116789, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.800711743772242, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8602430555555556, "step_test_time": 22.422787324001547, "step_plot_time": 13.474370372001431, "step_validation_cycle_time": 46.18120321300012} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_9000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..e0a84a322c53f66e9e6a3ce29876df5a06c510e6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 6.0, "train": {"Classification_binary": 0.26652294397354126, "Classification": 0.00040928192902356386, "Regression_pt": 0.34726932644844055, "Regression_eta": 0.032807424664497375, "Regression_sin_phi": 0.012679599225521088, "Regression_cos_phi": 0.006250202190130949, "Regression_energy": 0.3629995286464691, "Total": 2.3927316665649414, "Total_unweighted": 1.028938307077624}, "train_group": {}, "valid": {"Classification_binary": 0.2641235589981079, "Classification": 0.00047587649896740913, "Regression_pt": 0.3633097807566325, "Regression_eta": 0.05338614682356516, "Regression_sin_phi": 0.01317975545922915, "Regression_cos_phi": 0.013017172614733378, "Regression_energy": 0.3784254789352417, "Total": 3.1234445571899414, "Total_unweighted": 1.085917770086477, "metrics/particle/class_1/efficiency": 0.877728283405304, "metrics/particle/class_1/pid_accuracy": 0.985245406627655, "metrics/particle/class_2/efficiency": 0.05918663740158081, "metrics/particle/class_2/pid_accuracy": 0.0061349691823124886, "metrics/particle/class_3/efficiency": 0.37324050068855286, "metrics/particle/class_3/pid_accuracy": 0.955254316329956, "metrics/particle/class_4/efficiency": 0.7993197441101074, "metrics/particle/class_4/pid_accuracy": 0.04680851101875305, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.14912280440330505, "metrics/particle/count/bias_mean": -22.07942771911621, "metrics/particle/count/mae": 22.11328125, "metrics/particle/count/prediction_mean": 57.48698043823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3151704967021942, "metrics/particle/event/energy_response_mean": 0.6879472136497498, "metrics/particle/event/met_abs_error": 15.114838600158691, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2663697600364685, "metrics/particle/event/scalar_pt_response_mean": 0.7386794686317444, "metrics/particle/event/vector_pt_closure": 0.11548011749982834, "metrics/particle/matched/delta_eta_abs_mean": 0.008823337033390999, "metrics/particle/matched/delta_phi_abs_mean": 0.004129669163376093, "metrics/particle/matched/delta_r_mean": 0.01068991981446743, "metrics/particle/matched/energy_relative_abs_mean": 0.08512409776449203, "metrics/particle/matched/pid_accuracy": 0.9650494456291199, "metrics/particle/matched/pt_relative_abs_mean": 0.07813864201307297, "metrics/particle/matching/duplicate_fraction": 0.02663646638393402, "metrics/particle/matching/efficiency": 0.6236764788627625, "metrics/particle/matching/f1": 0.7241513729095459, "metrics/particle/matching/purity": 0.8632162809371948, "metrics/particle/matching/target_coverage_dr0p05": 0.6271622180938721, "metrics/particle/matching/target_coverage_dr0p10": 0.6501219272613525, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5543555021286011, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6435269117355347}, "step_train_time": 1.503688097000122, "step_valid_time": 9.754493117000038, "step_total_time": 2461.1157422065735, "elapsed_training_loop_seconds": 2461.1157422065735, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8049317459051964, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2739905400455389, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 582, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 511, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 460, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7363112391930836, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8780068728522337, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8009404388714734, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12199312714776632, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6628242074927954, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7903780068728522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7210031347962382, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7363112391930836, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8551199710886199, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2628137172982563, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1455, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1324, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1212, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8608582574772432, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9099656357388316, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8847310390912129, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09003436426116838, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.788036410923277, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8329896907216495, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8098897427330438, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8608582574772432, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8209871435716624, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2825716623966117, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1068, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 944, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 835, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8194444444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8838951310861424, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8504504504504504, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11610486891385768, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7248263888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7818352059925093, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7522522522522522, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8194444444444444, "step_test_time": 20.963837455999965, "step_plot_time": 12.785775686000306, "step_validation_cycle_time": 43.95999656799995} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..26378ebfb4f3236953de5ad69448af56474a916c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "idea", "data_dir": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 30, "checkpoint_freq": 2000, "val_freq": 1000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", "relaxed_load": false, "load_weights_only": true, "reset_task_loss_weights": true, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"idea": {"physical": {"batch_size": 1, "weight": null, "samples": {"idea_edm_qq_pf": {"version": "0.2.1", "splits": ["1"], "batch_size": null}, "idea_edm_ttbar_pf": {"version": "0.2.1", "splits": ["1"], "batch_size": null}, "idea_edm_ww_fullhad_pf": {"version": "0.2.1", "splits": ["1"], "batch_size": null}}}}}, "valid_dataset": {"idea": {"physical": {"batch_size": 1, "weight": null, "samples": {"idea_edm_qq_pf": {"version": "0.2.1", "splits": ["1"], "batch_size": null}, "idea_edm_ttbar_pf": {"version": "0.2.1", "splits": ["1"], "batch_size": null}, "idea_edm_ww_fullhad_pf": {"version": "0.2.1", "splits": ["1"], "batch_size": null}}}}}, "test_dataset": {"idea_edm_qq_pf": {"version": "0.2.1", "splits": ["1"], "batch_size": 1}, "idea_edm_ttbar_pf": {"version": "0.2.1", "splits": ["1"], "batch_size": 1}, "idea_edm_ww_fullhad_pf": {"version": "0.2.1", "splits": ["1"], "batch_size": 1}}, "enabled_test_datasets": ["idea_edm_qq_pf", "idea_edm_ttbar_pf", "idea_edm_ww_fullhad_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..701392f3a438b77f321d9a769e1424948a5ef132 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d39f3b6ad1ba83808af2eb434279dcdb07b7a28a5550fb88b8e90d22fd78bca +size 4933 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3a66a0224c31829b51ce12a50b7b6c00a91839f5 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..48821b874e601d3f4063f759930512cccae52ede Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..cf9cf1983acca6de50b6fb4d352ae6ac7734857f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..e3f3ca1d021a9c6a932fc3211ace0271cbeeeea0 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7d200ddd90ba318864f3230b26831c9e80fc34f2 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..7b05184c008bbf6ca015233e3064b8ff37adc0fa Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..03f14e2eac63bd08e4fe9585f04290c22249fe51 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..eea5bd06aec848d365828cc02260be373b9e7ef7 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7cda916dd3305713849faec9f92d4022840f32e6 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..bd0fc067a9d3de3a30ac19a4c93f773910018f2c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..535cd7b25472d1a623b7fef2a1a27a543449d4db Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..b40aff0c8b7743901b90494adf65d06a4af7d3ab Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_qq_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2b0a612b85f18636540d9e564cbfeec5cdb657fe Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..db28921e6d8b7b891413aeadec10780730458d28 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..785d07e3e5323f384154a5518c2c5f52d239a605 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..71f9b2a43fcb2a3add24fb05c33b891fab532e87 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0be67642e52ae3ac5faf36e295fdc91f625f51c2 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..0e548311bc1766c8438ffccc7d6b45db290c287b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b069e29babf7a6a66327821eef0a8266083830c0 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..cc5b319696da49fd48c4a89f676e9537fdab1d10 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..99898744e28ed09221ccf3dd08ed5f9189db5769 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..01e7963977562e001c975d2fe44ac1fdc2573c54 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1868443b84de4a7f19256321fbe8f90ef69c2ffd Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..86d6711f562cc8d185aac799a0982ffda0c10754 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b82186dcdc832a6d979b1c6ca41726352d13e9d4 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..f175f834c486d0f3548c3a99ec207e620ef6008e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..652dc6c8322de5798557bf0a70117f083b5fdd25 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..8774ca199c378fcf1477ceb9ffdb545e1e157319 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0ffcbed66feec45a8006e2a1c01b377287607af6 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..b44d56a075f8b76070756746718f71b7241b65ed Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5e9a3ab6ff9c61b3fbc93e0653a6828dac4cba16 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..e7d1d093501a7e5bc144a21545c5ab0d350d8641 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e09270ddeedecaaec3342bf6f90504e87d4ea6d0 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..589406e515911c1ce2f3fbc73b3a7faab165c71c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..65908e38e310b4432bd7931d86ba0992a9703864 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..fdd9f1b28dfd155575e88f30dd9372695ab5ebc9 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/plots_step_89000/idea_edm_ww_fullhad_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/runs/train/events.out.tfevents.1794727136.workergpu158.31729.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/runs/train/events.out.tfevents.1794727136.workergpu158.31729.0 new file mode 100644 index 0000000000000000000000000000000000000000..56c841db9762c053569b7d9828a0726a8d563f7f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/runs/train/events.out.tfevents.1794727136.workergpu158.31729.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cfda3e84a68cb659a4096e63aee1ffd500c9ee355d7b6e76252f3b8b944e483 +size 3286713 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/runs/valid/events.out.tfevents.1794727136.workergpu158.31729.1 b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/runs/valid/events.out.tfevents.1794727136.workergpu158.31729.1 new file mode 100644 index 0000000000000000000000000000000000000000..a55d2bf29877ef414bfc92ed6a739a807c513d1f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/runs/valid/events.out.tfevents.1794727136.workergpu158.31729.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d83030778b5037840131ac86a7ef0a273cec8ddeb31abcc4944b9b94c8e6af6e +size 44982314 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..6170ce0b62a05da202009ecf0c770c4c80c6dd84 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/scenario-manifest.json @@ -0,0 +1,1240 @@ +{ + "scenario": { + "name": "pf_fm_finetuning", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all" + }, + "allowed_variant_differences": [ + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets", + "data_config" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_finetuning", + "platform_name": "flatiron_h100_single", + "variant_name": "idea", + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "data_dir": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all", + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "idea", + "data_dir": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 30, + "checkpoint_freq": 2000, + "val_freq": 1000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "relaxed_load": false, + "load_weights_only": true, + "reset_task_loss_weights": true, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "idea": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "idea_edm_qq_pf": { + "version": "0.2.1", + "splits": [ + "1" + ], + "batch_size": null + }, + "idea_edm_ttbar_pf": { + "version": "0.2.1", + "splits": [ + "1" + ], + "batch_size": null + }, + "idea_edm_ww_fullhad_pf": { + "version": "0.2.1", + "splits": [ + "1" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "idea": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "idea_edm_qq_pf": { + "version": "0.2.1", + "splits": [ + "1" + ], + "batch_size": null + }, + "idea_edm_ttbar_pf": { + "version": "0.2.1", + "splits": [ + "1" + ], + "batch_size": null + }, + "idea_edm_ww_fullhad_pf": { + "version": "0.2.1", + "splits": [ + "1" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "idea_edm_qq_pf": { + "version": "0.2.1", + "splits": [ + "1" + ], + "batch_size": 1 + }, + "idea_edm_ttbar_pf": { + "version": "0.2.1", + "splits": [ + "1" + ], + "batch_size": 1 + }, + "idea_edm_ww_fullhad_pf": { + "version": "0.2.1", + "splits": [ + "1" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "idea_edm_qq_pf", + "idea_edm_ttbar_pf", + "idea_edm_ww_fullhad_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-idea-pipeline-v1", + "--production-name", + "idea", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--load", + "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "1000", + "--checkpoint_freq", + "2000", + "--patience", + "30", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--load_weights_only", + "true", + "--reset_task_loss_weights", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.feature_bag", + "true", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.trainable", + "all", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3232f023ae294f753a835a9a242de99f746eee3f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/train-config.yaml @@ -0,0 +1,207 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds +dataset: idea +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- idea_edm_qq_pf +- idea_edm_ttbar_pf +- idea_edm_ww_fullhad_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +load_weights_only: true +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 30 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: true +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + idea_edm_qq_pf: + batch_size: 1 + splits: + - '1' + version: 0.2.1 + idea_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + version: 0.2.1 + idea_edm_ww_fullhad_pf: + batch_size: 1 + splits: + - '1' + version: 0.2.1 +threads: 16 +train: true +train_dataset: + idea: + physical: + batch_size: 1 + samples: + idea_edm_qq_pf: + batch_size: null + splits: + - '1' + version: 0.2.1 + idea_edm_ttbar_pf: + batch_size: null + splits: + - '1' + version: 0.2.1 + idea_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + version: 0.2.1 + weight: null +val_freq: 1000 +valid_dataset: + idea: + physical: + batch_size: 1 + samples: + idea_edm_qq_pf: + batch_size: null + splits: + - '1' + version: 0.2.1 + idea_edm_ttbar_pf: + batch_size: null + splits: + - '1' + version: 0.2.1 + idea_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + version: 0.2.1 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/train.log b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1ff2d673fa1f2aee7ad34f1344029306674d3225 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/train.log @@ -0,0 +1,10820 @@ +rank0 [2026-11-15 02:18:52,613] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-15 02:18:52,613] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-15 02:18:53,736] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 02:18:53,736] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 02:18:53,737] INFO : Instantiating model +rank0 [2026-11-15 02:18:53,774] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 02:18:53,775] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 02:18:53,776] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 02:18:53,776] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 02:18:53,776] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 02:18:53,776] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 02:18:53,777] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,779] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 02:18:53,779] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,782] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 02:18:53,782] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,784] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 02:18:53,785] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,787] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 02:18:53,787] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,790] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 02:18:53,790] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,792] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 02:18:53,793] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 02:18:53,793] INFO : Initializing output DNNs +rank0 [2026-11-15 02:18:53,807] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,810] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,817] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-15 02:18:53,817] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 02:18:53,817] INFO : jagged_attention enabled=True +rank0 [2026-11-15 02:18:53,817] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 02:18:53,817] INFO : task_queries enabled=True +rank0 [2026-11-15 02:18:53,817] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 02:18:53,817] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 02:18:53,817] INFO : classification_query parameters: 256 +rank0 [2026-11-15 02:18:53,817] INFO : regression_query parameters: 256 +rank0 [2026-11-15 02:18:53,817] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 02:18:53,817] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 02:18:53,817] INFO : output_mode=elementwise +rank0 [2026-11-15 02:18:53,817] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 02:18:53,817] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 02:18:53,817] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 02:18:53,817] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 02:18:53,818] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 02:18:53,818] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 02:18:53,818] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 02:18:53,818] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 02:18:53,819] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 02:18:53,819] INFO : MLPF __init__ done +rank0 [2026-11-15 02:18:53,819] INFO : Instantiated model +rank0 [2026-11-15 02:18:53,820] INFO : Moving model to device rank=0 +rank0 [2026-11-15 02:18:53,831] INFO : Moved model to device rank=0 +rank0 [2026-11-15 02:18:53,832] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 02:18:53,963] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 02:18:53,963] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 02:18:53,963] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 02:18:54,062] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 02:18:54,062] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 02:18:54,064] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 02:18:54,064] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 02:18:54,064] INFO : Total parameters: 3910930 +rank0 [2026-11-15 02:18:54,066] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 02:18:54,071] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856 +rank0 [2026-11-15 02:18:54,071] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856 +rank0 [2026-11-15 02:18:54,071] INFO : Getting dataloaders +rank0 [2026-11-15 02:18:54,072] INFO : sample=idea_edm_qq_pf split=train split_configs=['1'] +rank0 [2026-11-15 02:18:56,127] INFO : train_dataset: idea_edm_qq_pf, 8000 +rank0 [2026-11-15 02:18:56,127] INFO : sample=idea_edm_ttbar_pf split=train split_configs=['1'] +rank0 [2026-11-15 02:18:56,171] INFO : train_dataset: idea_edm_ttbar_pf, 8000 +rank0 [2026-11-15 02:18:56,171] INFO : sample=idea_edm_ww_fullhad_pf split=train split_configs=['1'] +rank0 [2026-11-15 02:18:56,214] INFO : train_dataset: idea_edm_ww_fullhad_pf, 8000 +rank0 [2026-11-15 02:18:56,215] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 02:18:56,215] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 02:18:56,215] INFO : train mix: physical: weight=1500 step_share=1.000 batches_per_epoch=1500 events_per_step=16 expected_passes_in_500000_steps=333.33 +rank0 [2026-11-15 02:18:56,215] INFO : sample=idea_edm_qq_pf split=valid split_configs=['1'] +rank0 [2026-11-15 02:18:56,223] INFO : valid_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:18:56,223] INFO : sample=idea_edm_ttbar_pf split=valid split_configs=['1'] +rank0 [2026-11-15 02:18:56,231] INFO : valid_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:18:56,231] INFO : sample=idea_edm_ww_fullhad_pf split=valid split_configs=['1'] +rank0 [2026-11-15 02:18:56,240] INFO : valid_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:18:56,240] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 02:18:56,240] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 02:18:56,294] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:18:56,382] INFO : Got dataloaders +rank0 [2026-11-15 02:18:56,383] INFO : loader split=train rank=0 len=1500 +rank0 [2026-11-15 02:18:56,383] INFO : loader split=valid rank=0 len=48 +rank0 [2026-11-15 02:18:56,383] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 02:18:56,392] INFO : Creating train_iterator +rank0 [2026-11-15 02:18:56,392] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:18:56,412] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:18:56,519] INFO : Created train_iterator +rank0 [2026-11-15 02:19:21,499] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9513028985261918, 'Classification': 0.007238105600699782, 'Regression_pt': 3.262555984258652, 'Regression_eta': 0.15020455058664084, 'Regression_sin_phi': 0.031022030971944333, 'Regression_cos_phi': 0.03321564427576959, 'Regression_energy': 3.2605306005477903}, weights={'Classification_binary': 1.0, 'Classification': 269.5875244140625, 'Regression_pt': 0.5980902314186096, 'Regression_eta': 12.990970611572266, 'Regression_sin_phi': 62.90055465698242, 'Regression_cos_phi': 58.74650192260742, 'Regression_energy': 0.5984617471694946} +rank0 [2026-11-15 02:22:52,456] INFO : Step 1000/500000 rank0 | Train Loss: 4.6716 | Train Loss Unweighted: 1.8902 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2859 | Regression_cos_phi: 0.0147 | Regression_energy: 0.7599 | Regression_eta: 0.0895 | Regression_pt: 0.7188 | Regression_sin_phi: 0.0208 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1668 | Classification_binary: 0.2859 | Regression_cos_phi: 0.8650 | Regression_energy: 0.4548 | Regression_eta: 1.1624 | Regression_pt: 0.4299 | Regression_sin_phi: 1.3069 | LR: 2.01e-05 | DataLoad Time: 0.0018s | Model Forward Time: 1.2127s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0026 | max_forward: 0.0682 | max_loss: 0.0030 | max_backward: 0.1429 | max_optimizer_step: 0.0569 | max_tensorboard_logging: 0.7698 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.6 | uss_MB: 1351.5 | child_rss_MB: 23464.4 | allocated_MB: 199.6 | reserved_MB: 23934.0 | inactive_split_MB: 486.4 +rank0 [2026-11-15 02:22:52,457] INFO : Running validation on rank0 +rank0 [2026-11-15 02:22:52,458] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:22:52,458] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:22:53,109] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 02:22:53,269] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:22:53,347] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:22:53,401] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:22:53,455] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:22:53,501] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:22:53,555] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:22:53,611] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:22:53,659] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:22:53,727] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:22:53,795] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:22:53,850] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:22:53,905] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:22:53,953] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:22:54,008] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:22:54,063] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:22:54,111] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:22:54,116] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:22:54,119] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:22:54,121] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:22:54,124] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:22:54,127] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:22:54,129] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:22:54,132] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:22:54,156] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:22:54,159] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:22:54,211] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:22:54,214] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:22:54,216] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:22:54,219] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:22:54,222] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:22:54,224] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:23:01,883] INFO : VALIDATION | Step=1000/500000 | Train Loss=4.6716 | Valid Loss=4.2833 | Train Loss Unweighted=1.8902 | Valid Loss Unweighted=1.5921 | Stale=0 | ETA=2041.7m +rank0 [2026-11-15 02:23:01,883] INFO : split_configs=['1'] +rank0 [2026-11-15 02:23:01,895] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:23:01,901] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:23:01,964] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:02,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:03,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:03,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:03,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:04,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:04,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:04,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:05,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:05,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:05,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:06,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:06,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:06,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:07,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:07,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:07,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:07,820] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 02:23:07,822] INFO : split_configs=['1'] +rank0 [2026-11-15 02:23:07,833] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:23:07,837] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:23:07,873] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:08,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:09,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:09,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:10,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:10,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:10,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:11,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:11,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:12,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:12,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:13,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:13,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:13,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:14,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:14,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:15,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:15,324] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:23:15,327] INFO : split_configs=['1'] +rank0 [2026-11-15 02:23:15,337] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:23:15,342] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:23:15,374] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:16,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:16,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:16,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:17,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:17,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:18,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:18,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:19,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:19,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:19,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:20,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:20,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:21,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:21,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:21,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:22,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:22,350] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:23:22,358] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf +rank0 [2026-11-15 02:23:23,488] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf +rank0 [2026-11-15 02:23:23,977] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:25,650] INFO : Plotted jets +rank0 [2026-11-15 02:23:26,666] INFO : Plotted jet ratio +rank0 [2026-11-15 02:23:26,678] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:23:28,262] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:23:28,703] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:30,052] INFO : Plotted jets +rank0 [2026-11-15 02:23:31,077] INFO : Plotted jet ratio +rank0 [2026-11-15 02:23:31,090] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:23:32,492] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:23:32,931] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:34,241] INFO : Plotted jets +rank0 [2026-11-15 02:23:35,220] INFO : Plotted jet ratio +rank0 [2026-11-15 02:25:31,934] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 02:25:31,934] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:28,686] INFO : Step 2000/500000 rank0 | Train Loss: 4.2064 | Train Loss Unweighted: 1.3477 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2688 | Regression_cos_phi: 0.0168 | Regression_energy: 0.4710 | Regression_eta: 0.1028 | Regression_pt: 0.4738 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1751 | Classification_binary: 0.2688 | Regression_cos_phi: 0.9847 | Regression_energy: 0.2818 | Regression_eta: 1.3358 | Regression_pt: 0.2834 | Regression_sin_phi: 0.8768 | LR: 2.02e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.2354s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0690 | max_loss: 0.0029 | max_backward: 0.1503 | max_optimizer_step: 0.0604 | max_tensorboard_logging: 0.7507 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 4841.1 | uss_MB: 3619.9 | child_rss_MB: 23751.4 | allocated_MB: 197.5 | reserved_MB: 19548.0 | inactive_split_MB: 136.5 +rank0 [2026-11-15 02:27:28,688] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 02:27:28,752] INFO : Running validation on rank0 +rank0 [2026-11-15 02:27:28,752] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:27:28,752] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:27:29,363] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 02:27:29,481] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:27:29,593] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:27:29,660] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:27:29,725] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:27:29,789] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:27:29,854] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:27:29,918] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:27:30,023] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:27:30,125] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:27:30,224] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:27:30,290] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:27:30,354] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:27:30,417] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:27:30,482] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:27:30,546] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:27:30,645] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:27:30,664] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:27:30,677] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:27:30,680] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:27:30,690] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:27:30,703] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:27:30,706] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:27:30,721] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:27:30,724] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:27:30,726] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:27:30,728] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:27:30,732] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:27:30,747] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:27:30,749] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:27:30,752] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:27:30,754] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:27:38,443] INFO : VALIDATION | Step=2000/500000 | Train Loss=4.2064 | Valid Loss=3.8027 | Train Loss Unweighted=1.3477 | Valid Loss Unweighted=1.3612 | Stale=0 | ETA=2166.5m +rank0 [2026-11-15 02:27:38,443] INFO : split_configs=['1'] +rank0 [2026-11-15 02:27:38,454] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:27:38,461] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:27:38,528] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:39,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:39,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:39,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:40,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:40,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:40,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:41,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:41,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:41,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:42,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:42,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:42,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:43,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:43,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:43,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:44,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:44,338] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:27:44,341] INFO : split_configs=['1'] +rank0 [2026-11-15 02:27:44,351] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:27:44,355] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:27:44,391] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:45,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:45,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:46,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:46,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:46,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:47,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:47,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:48,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:48,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:49,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:49,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:50,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:50,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:50,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:51,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:51,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:51,897] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:27:51,901] INFO : split_configs=['1'] +rank0 [2026-11-15 02:27:51,912] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:27:51,916] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:27:51,954] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:52,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:53,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:53,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:53,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:54,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:54,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:55,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:55,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:56,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:56,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:56,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:57,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:57,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:58,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:58,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:58,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:58,948] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:27:58,956] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf +rank0 [2026-11-15 02:28:00,019] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf +rank0 [2026-11-15 02:28:00,453] INFO : Plotted number of elements +rank0 [2026-11-15 02:28:02,127] INFO : Plotted jets +rank0 [2026-11-15 02:28:03,143] INFO : Plotted jet ratio +rank0 [2026-11-15 02:28:03,156] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:28:04,663] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:28:05,117] INFO : Plotted number of elements +rank0 [2026-11-15 02:28:06,435] INFO : Plotted jets +rank0 [2026-11-15 02:28:07,442] INFO : Plotted jet ratio +rank0 [2026-11-15 02:28:07,452] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:28:08,858] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:28:09,317] INFO : Plotted number of elements +rank0 [2026-11-15 02:28:10,609] INFO : Plotted jets +rank0 [2026-11-15 02:28:11,593] INFO : Plotted jet ratio +rank0 [2026-11-15 02:32:06,351] INFO : Step 3000/500000 rank0 | Train Loss: 5.3811 | Train Loss Unweighted: 1.5084 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2532 | Regression_cos_phi: 0.0121 | Regression_energy: 0.5437 | Regression_eta: 0.2140 | Regression_pt: 0.4705 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1254 | Classification_binary: 0.2532 | Regression_cos_phi: 0.7113 | Regression_energy: 0.3254 | Regression_eta: 2.7795 | Regression_pt: 0.2814 | Regression_sin_phi: 0.9049 | LR: 2.05e-05 | DataLoad Time: 0.0015s | Model Forward Time: 1.2663s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0614 | max_loss: 0.0026 | max_backward: 0.1330 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.7979 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5418.4 | uss_MB: 4197.4 | child_rss_MB: 23862.1 | allocated_MB: 198.0 | reserved_MB: 16382.0 | inactive_split_MB: 172.0 +rank0 [2026-11-15 02:32:06,356] INFO : Running validation on rank0 +rank0 [2026-11-15 02:32:06,356] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:32:06,356] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:32:06,976] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 02:32:07,095] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:32:07,211] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:32:07,280] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:32:07,345] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:32:07,411] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:32:07,477] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:32:07,543] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:32:07,649] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:32:07,751] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:32:07,851] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:32:07,919] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:32:07,984] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:32:08,049] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:32:08,116] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:32:08,181] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:32:08,283] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:32:08,306] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:32:08,319] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:32:08,322] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:32:08,333] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:32:08,365] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:32:08,382] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:32:08,383] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:32:08,385] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:32:08,386] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:32:08,387] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:32:08,396] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:32:08,397] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:32:08,398] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:32:08,399] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:32:08,400] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:32:16,091] INFO : VALIDATION | Step=3000/500000 | Train Loss=5.3811 | Valid Loss=3.5861 | Train Loss Unweighted=1.5084 | Valid Loss Unweighted=1.2585 | Stale=0 | ETA=2208.1m +rank0 [2026-11-15 02:32:16,092] INFO : split_configs=['1'] +rank0 [2026-11-15 02:32:16,103] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:32:16,116] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:32:16,182] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:32:16,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:32:17,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:32:17,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:32:17,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:32:18,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:32:18,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:32:18,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:32:19,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:32:19,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:32:19,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:32:20,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:32:20,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:32:20,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:32:21,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:32:21,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:32:21,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:32:21,942] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:32:21,945] INFO : split_configs=['1'] +rank0 [2026-11-15 02:32:21,956] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:32:21,960] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:32:22,000] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:32:22,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:32:23,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:32:23,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:32:24,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:32:24,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:32:25,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:32:25,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:32:25,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:32:26,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:32:26,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:32:27,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:32:27,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:32:28,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:32:28,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:32:29,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:32:29,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:32:29,567] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:32:29,571] INFO : split_configs=['1'] +rank0 [2026-11-15 02:32:29,582] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:32:29,586] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:32:29,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:32:30,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:32:30,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:32:31,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:32:31,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:32:32,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:32:32,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:32:32,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:32:33,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:32:33,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:32:34,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:32:34,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:32:34,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:32:35,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:32:35,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:32:36,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:32:36,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:32:36,587] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:32:36,595] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf +rank0 [2026-11-15 02:32:37,689] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf +rank0 [2026-11-15 02:32:38,127] INFO : Plotted number of elements +rank0 [2026-11-15 02:32:39,800] INFO : Plotted jets +rank0 [2026-11-15 02:32:40,829] INFO : Plotted jet ratio +rank0 [2026-11-15 02:32:40,842] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:32:42,356] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:32:42,795] INFO : Plotted number of elements +rank0 [2026-11-15 02:32:44,109] INFO : Plotted jets +rank0 [2026-11-15 02:32:45,125] INFO : Plotted jet ratio +rank0 [2026-11-15 02:32:45,138] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:32:46,512] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:32:46,944] INFO : Plotted number of elements +rank0 [2026-11-15 02:32:48,243] INFO : Plotted jets +rank0 [2026-11-15 02:32:49,216] INFO : Plotted jet ratio +rank0 [2026-11-15 02:32:49,585] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 02:32:49,585] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:36:43,273] INFO : Step 4000/500000 rank0 | Train Loss: 3.2529 | Train Loss Unweighted: 1.2183 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2756 | Regression_cos_phi: 0.0109 | Regression_energy: 0.4325 | Regression_eta: 0.0706 | Regression_pt: 0.4162 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1572 | Classification_binary: 0.2756 | Regression_cos_phi: 0.6375 | Regression_energy: 0.2588 | Regression_eta: 0.9176 | Regression_pt: 0.2489 | Regression_sin_phi: 0.7573 | LR: 2.08e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.3199s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0026 | max_forward: 0.0650 | max_loss: 0.0029 | max_backward: 0.1395 | max_optimizer_step: 0.0639 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0006 | Memory: rss_MB: 5603.4 | uss_MB: 4382.4 | child_rss_MB: 23919.4 | allocated_MB: 200.6 | reserved_MB: 19108.0 | inactive_split_MB: 225.4 +rank0 [2026-11-15 02:36:43,275] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 02:36:43,341] INFO : Running validation on rank0 +rank0 [2026-11-15 02:36:43,341] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:36:43,341] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:36:43,961] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 02:36:44,080] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:36:44,195] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:36:44,266] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:36:44,332] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:36:44,398] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:36:44,467] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:36:44,534] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:36:44,641] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:36:44,745] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:36:44,848] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:36:44,916] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:36:44,983] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:36:45,051] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:36:45,119] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:36:45,186] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:36:45,289] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:36:45,311] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:36:45,324] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:36:45,325] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:36:45,362] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:36:45,375] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:36:45,378] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:36:45,381] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:36:45,386] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:36:45,389] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:36:45,392] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:36:45,395] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:36:45,398] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:36:45,402] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:36:45,405] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:36:45,408] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:36:53,103] INFO : VALIDATION | Step=4000/500000 | Train Loss=3.2529 | Valid Loss=3.4666 | Train Loss Unweighted=1.2183 | Valid Loss Unweighted=1.2061 | Stale=0 | ETA=2225.2m +rank0 [2026-11-15 02:36:53,104] INFO : split_configs=['1'] +rank0 [2026-11-15 02:36:53,115] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:36:53,119] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:36:53,190] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:36:53,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:36:54,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:36:54,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:36:54,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:36:55,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:36:55,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:36:56,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:36:56,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:36:56,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:36:57,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:36:57,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:36:57,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:36:58,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:36:58,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:36:58,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:36:59,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:36:59,086] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:36:59,088] INFO : split_configs=['1'] +rank0 [2026-11-15 02:36:59,099] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:36:59,114] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:36:59,156] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:37:00,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:37:00,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:37:00,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:37:01,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:37:01,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:37:02,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:37:02,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:37:03,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:37:03,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:37:04,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:37:04,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:37:04,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:37:05,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:37:05,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:37:06,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:37:06,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:37:06,749] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:37:06,752] INFO : split_configs=['1'] +rank0 [2026-11-15 02:37:06,762] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:37:06,766] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:37:06,813] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:37:07,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:37:08,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:37:08,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:37:08,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:37:09,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:37:09,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:37:10,142] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:37:10,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:37:10,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:37:11,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:37:11,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:37:12,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:37:12,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:37:13,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:37:13,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:37:13,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:37:13,911] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:37:13,919] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf +rank0 [2026-11-15 02:37:15,007] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf +rank0 [2026-11-15 02:37:15,424] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:17,089] INFO : Plotted jets +rank0 [2026-11-15 02:37:18,119] INFO : Plotted jet ratio +rank0 [2026-11-15 02:37:18,129] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:37:19,647] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:37:20,102] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:21,427] INFO : Plotted jets +rank0 [2026-11-15 02:37:22,440] INFO : Plotted jet ratio +rank0 [2026-11-15 02:37:22,450] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:37:23,825] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:37:24,268] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:25,569] INFO : Plotted jets +rank0 [2026-11-15 02:37:26,924] INFO : Plotted jet ratio +rank0 [2026-11-15 02:39:23,160] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 02:39:23,160] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:41:20,459] INFO : Step 5000/500000 rank0 | Train Loss: 4.7778 | Train Loss Unweighted: 1.1399 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2611 | Regression_cos_phi: 0.0277 | Regression_energy: 0.3852 | Regression_eta: 0.0867 | Regression_pt: 0.3604 | Regression_sin_phi: 0.0182 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1731 | Classification_binary: 0.2611 | Regression_cos_phi: 1.6256 | Regression_energy: 0.2305 | Regression_eta: 1.1261 | Regression_pt: 0.2155 | Regression_sin_phi: 1.1457 | LR: 2.13e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.2270s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0626 | max_loss: 0.0029 | max_backward: 0.1312 | max_optimizer_step: 0.0639 | max_tensorboard_logging: 0.7448 | max_tensorboard_flush: 0.0008 | Memory: rss_MB: 5607.3 | uss_MB: 4386.2 | child_rss_MB: 23785.4 | allocated_MB: 195.3 | reserved_MB: 22866.0 | inactive_split_MB: 268.7 +rank0 [2026-11-15 02:41:20,462] INFO : Running validation on rank0 +rank0 [2026-11-15 02:41:20,462] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:41:20,462] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:41:21,083] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 02:41:21,199] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:41:21,314] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:41:21,383] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:41:21,450] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:41:21,516] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:41:21,584] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:41:21,653] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:41:21,769] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:41:21,875] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:41:21,980] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:41:22,052] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:41:22,120] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:41:22,187] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:41:22,255] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:41:22,322] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:41:22,424] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:41:22,434] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:41:22,437] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:41:22,441] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:41:22,445] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:41:22,448] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:41:22,451] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:41:22,454] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:41:22,458] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:41:22,461] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:41:22,465] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:41:22,468] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:41:22,471] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:41:22,475] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:41:22,478] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:41:22,481] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:41:30,189] INFO : VALIDATION | Step=5000/500000 | Train Loss=4.7778 | Valid Loss=3.4041 | Train Loss Unweighted=1.1399 | Valid Loss Unweighted=1.1713 | Stale=0 | ETA=2233.8m +rank0 [2026-11-15 02:41:30,190] INFO : split_configs=['1'] +rank0 [2026-11-15 02:41:30,201] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:41:30,208] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:41:30,284] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:31,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:31,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:31,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:32,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:32,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:32,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:33,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:33,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:33,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:34,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:34,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:34,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:35,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:35,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:35,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:36,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:36,110] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:41:36,113] INFO : split_configs=['1'] +rank0 [2026-11-15 02:41:36,124] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:41:36,128] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:41:36,175] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:37,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:37,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:37,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:38,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:38,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:39,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:39,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:40,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:40,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:41,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:41,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:41,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:42,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:42,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:43,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:43,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:43,713] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:41:43,716] INFO : split_configs=['1'] +rank0 [2026-11-15 02:41:43,727] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:41:43,731] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:41:43,774] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:44,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:45,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:45,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:45,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:46,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:46,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:47,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:47,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:47,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:48,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:48,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:49,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:49,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:49,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:50,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:50,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:50,777] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:41:50,784] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf +rank0 [2026-11-15 02:41:51,867] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf +rank0 [2026-11-15 02:41:52,290] INFO : Plotted number of elements +rank0 [2026-11-15 02:41:53,975] INFO : Plotted jets +rank0 [2026-11-15 02:41:54,982] INFO : Plotted jet ratio +rank0 [2026-11-15 02:41:54,993] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:41:56,523] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:41:56,964] INFO : Plotted number of elements +rank0 [2026-11-15 02:41:58,273] INFO : Plotted jets +rank0 [2026-11-15 02:41:59,295] INFO : Plotted jet ratio +rank0 [2026-11-15 02:41:59,304] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:42:00,701] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:42:01,140] INFO : Plotted number of elements +rank0 [2026-11-15 02:42:02,432] INFO : Plotted jets +rank0 [2026-11-15 02:42:03,409] INFO : Plotted jet ratio +rank0 [2026-11-15 02:45:57,687] INFO : Step 6000/500000 rank0 | Train Loss: 2.3339 | Train Loss Unweighted: 1.0330 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2483 | Regression_cos_phi: 0.0113 | Regression_energy: 0.3749 | Regression_eta: 0.0238 | Regression_pt: 0.3655 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1166 | Classification_binary: 0.2483 | Regression_cos_phi: 0.6613 | Regression_energy: 0.2244 | Regression_eta: 0.3093 | Regression_pt: 0.2186 | Regression_sin_phi: 0.5556 | LR: 2.19e-05 | DataLoad Time: 0.0023s | Model Forward Time: 1.8866s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0027 | max_forward: 0.0633 | max_loss: 0.0027 | max_backward: 0.1378 | max_optimizer_step: 0.0716 | max_tensorboard_logging: 1.3724 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5658.1 | uss_MB: 4437.1 | child_rss_MB: 23910.9 | allocated_MB: 199.4 | reserved_MB: 20068.0 | inactive_split_MB: 472.6 +rank0 [2026-11-15 02:45:57,691] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 02:45:57,759] INFO : Running validation on rank0 +rank0 [2026-11-15 02:45:57,759] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:45:57,759] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:45:58,381] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 02:45:58,503] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:45:58,618] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:45:58,690] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:45:58,759] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:45:58,829] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:45:58,898] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:45:58,968] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:45:59,069] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:45:59,172] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:45:59,276] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:45:59,343] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:45:59,411] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:45:59,478] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:45:59,545] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:45:59,613] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:45:59,715] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:45:59,723] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:45:59,725] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:45:59,726] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:45:59,727] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:45:59,728] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:45:59,729] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:45:59,730] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:45:59,732] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:45:59,733] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:45:59,734] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:45:59,750] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:45:59,751] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:45:59,752] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:45:59,753] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:45:59,754] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:46:07,574] INFO : VALIDATION | Step=6000/500000 | Train Loss=2.3339 | Valid Loss=3.3140 | Train Loss Unweighted=1.0330 | Valid Loss Unweighted=1.1478 | Stale=0 | ETA=2238.3m +rank0 [2026-11-15 02:46:07,576] INFO : split_configs=['1'] +rank0 [2026-11-15 02:46:07,586] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:46:07,589] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:46:07,658] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:46:08,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:46:08,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:46:09,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:46:09,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:46:09,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:46:10,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:46:10,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:46:10,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:46:11,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:46:11,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:46:11,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:46:12,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:46:12,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:46:12,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:46:13,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:46:13,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:46:13,583] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 02:46:13,586] INFO : split_configs=['1'] +rank0 [2026-11-15 02:46:13,597] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:46:13,601] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:46:13,640] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:46:14,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:46:15,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:46:15,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:46:15,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:46:16,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:46:16,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:46:17,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:46:17,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:46:18,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:46:18,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:46:19,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:46:19,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:46:19,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:46:20,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:46:20,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:46:21,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:46:21,307] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:46:21,311] INFO : split_configs=['1'] +rank0 [2026-11-15 02:46:21,322] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:46:21,325] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:46:21,365] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:46:22,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:46:22,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:46:23,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:46:23,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:46:23,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:46:24,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:46:24,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:46:25,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:46:25,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:46:25,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:46:26,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:46:26,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:46:27,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:46:27,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:46:27,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:46:28,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:46:28,348] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:46:28,358] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf +rank0 [2026-11-15 02:46:29,458] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf +rank0 [2026-11-15 02:46:29,896] INFO : Plotted number of elements +rank0 [2026-11-15 02:46:31,564] INFO : Plotted jets +rank0 [2026-11-15 02:46:32,579] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:32,591] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:46:34,149] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:46:34,593] INFO : Plotted number of elements +rank0 [2026-11-15 02:46:35,911] INFO : Plotted jets +rank0 [2026-11-15 02:46:36,932] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:36,941] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:46:38,337] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:46:38,786] INFO : Plotted number of elements +rank0 [2026-11-15 02:46:40,093] INFO : Plotted jets +rank0 [2026-11-15 02:46:41,077] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:41,446] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 02:46:41,447] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:50:34,377] INFO : Step 7000/500000 rank0 | Train Loss: 3.5359 | Train Loss Unweighted: 1.2333 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.2855 | Regression_cos_phi: 0.0191 | Regression_energy: 0.4518 | Regression_eta: 0.0362 | Regression_pt: 0.4250 | Regression_sin_phi: 0.0152 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1795 | Classification_binary: 0.2855 | Regression_cos_phi: 1.1232 | Regression_energy: 0.2704 | Regression_eta: 0.4700 | Regression_pt: 0.2542 | Regression_sin_phi: 0.9532 | LR: 2.26e-05 | DataLoad Time: 0.0020s | Model Forward Time: 1.4969s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0593 | max_loss: 0.0029 | max_backward: 0.1280 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 1.0013 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5743.9 | uss_MB: 4522.9 | child_rss_MB: 23824.1 | allocated_MB: 196.8 | reserved_MB: 18104.0 | inactive_split_MB: 351.2 +rank0 [2026-11-15 02:50:34,381] INFO : Running validation on rank0 +rank0 [2026-11-15 02:50:34,381] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:50:34,381] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:50:34,997] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 02:50:35,118] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:50:35,234] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:50:35,304] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:50:35,373] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:50:35,440] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:50:35,507] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:50:35,574] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:50:35,675] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:50:35,779] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:50:35,880] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:50:35,949] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:50:36,016] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:50:36,083] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:50:36,150] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:50:36,217] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:50:36,321] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:50:36,343] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:50:36,356] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:50:36,360] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:50:36,381] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:50:36,384] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:50:36,385] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:50:36,386] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:50:36,388] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:50:36,388] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:50:36,389] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:50:36,391] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:50:36,392] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:50:36,402] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:50:36,403] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:50:36,404] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:50:44,096] INFO : VALIDATION | Step=7000/500000 | Train Loss=3.5359 | Valid Loss=3.2511 | Train Loss Unweighted=1.2333 | Valid Loss Unweighted=1.1241 | Stale=0 | ETA=2239.3m +rank0 [2026-11-15 02:50:44,097] INFO : split_configs=['1'] +rank0 [2026-11-15 02:50:44,134] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:50:44,139] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:50:44,216] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:50:44,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:50:45,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:50:45,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:50:46,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:50:46,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:50:46,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:50:47,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:50:47,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:50:47,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:50:48,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:50:48,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:50:48,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:50:49,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:50:49,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:50:49,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:50:49,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:50:50,052] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:50:50,055] INFO : split_configs=['1'] +rank0 [2026-11-15 02:50:50,101] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:50:50,104] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:50:50,148] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:50:51,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:50:51,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:50:51,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:50:52,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:50:52,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:50:53,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:50:53,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:50:54,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:50:54,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:50:55,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:50:55,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:50:55,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:50:56,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:50:56,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:50:57,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:50:57,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:50:57,769] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:50:57,772] INFO : split_configs=['1'] +rank0 [2026-11-15 02:50:57,806] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:50:57,811] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:50:57,857] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:50:58,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:50:59,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:50:59,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:50:59,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:51:00,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:51:00,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:51:01,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:51:01,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:51:02,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:51:02,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:51:02,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:51:03,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:51:03,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:51:04,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:51:04,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:51:04,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:51:04,940] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:51:04,951] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf +rank0 [2026-11-15 02:51:06,047] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf +rank0 [2026-11-15 02:51:06,470] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:08,141] INFO : Plotted jets +rank0 [2026-11-15 02:51:09,158] INFO : Plotted jet ratio +rank0 [2026-11-15 02:51:09,170] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:51:10,716] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:51:11,159] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:12,468] INFO : Plotted jets +rank0 [2026-11-15 02:51:13,480] INFO : Plotted jet ratio +rank0 [2026-11-15 02:51:13,490] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:51:14,869] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:51:15,313] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:16,609] INFO : Plotted jets +rank0 [2026-11-15 02:51:17,588] INFO : Plotted jet ratio +rank0 [2026-11-15 02:53:14,077] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 02:53:14,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:55:11,046] INFO : Step 8000/500000 rank0 | Train Loss: 3.4353 | Train Loss Unweighted: 1.2436 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2849 | Regression_cos_phi: 0.0204 | Regression_energy: 0.4547 | Regression_eta: 0.0368 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1502 | Classification_binary: 0.2849 | Regression_cos_phi: 1.1979 | Regression_energy: 0.2721 | Regression_eta: 0.4782 | Regression_pt: 0.2594 | Regression_sin_phi: 0.7925 | LR: 2.34e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.3671s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0584 | max_loss: 0.0029 | max_backward: 0.1237 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.8720 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5934.2 | uss_MB: 4713.2 | child_rss_MB: 23732.5 | allocated_MB: 199.5 | reserved_MB: 18154.0 | inactive_split_MB: 174.5 +rank0 [2026-11-15 02:55:11,048] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 02:55:11,116] INFO : Running validation on rank0 +rank0 [2026-11-15 02:55:11,116] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:55:11,116] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:55:11,729] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 02:55:11,846] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:55:11,963] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:55:12,035] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:55:12,103] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:55:12,171] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:55:12,240] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:55:12,309] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:55:12,414] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:55:12,518] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:55:12,620] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:55:12,689] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:55:12,757] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:55:12,823] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:55:12,891] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:55:12,958] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:55:13,061] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:55:13,071] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:55:13,075] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:55:13,078] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:55:13,082] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:55:13,085] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:55:13,089] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:55:13,092] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:55:13,095] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:55:13,099] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:55:13,102] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:55:13,105] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:55:13,108] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:55:13,111] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:55:13,115] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:55:13,119] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:55:20,819] INFO : VALIDATION | Step=8000/500000 | Train Loss=3.4353 | Valid Loss=3.2100 | Train Loss Unweighted=1.2436 | Valid Loss Unweighted=1.1045 | Stale=0 | ETA=2239.0m +rank0 [2026-11-15 02:55:20,821] INFO : split_configs=['1'] +rank0 [2026-11-15 02:55:20,831] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:55:20,838] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:55:20,917] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:21,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:22,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:22,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:22,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:23,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:23,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:23,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:24,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:24,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:24,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:25,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:25,429] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:25,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:26,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:26,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:26,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:26,766] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:55:26,769] INFO : split_configs=['1'] +rank0 [2026-11-15 02:55:26,779] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:55:26,783] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:55:26,828] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:27,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:28,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:28,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:29,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:29,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:29,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:30,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:30,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:31,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:31,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:32,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:32,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:33,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:33,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:33,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:34,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:34,430] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:55:34,434] INFO : split_configs=['1'] +rank0 [2026-11-15 02:55:34,468] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:55:34,471] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:55:34,518] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:35,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:35,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:36,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:36,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:37,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:37,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:37,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:38,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:38,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:39,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:39,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:39,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:40,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:40,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:41,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:41,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:41,623] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:55:41,630] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf +rank0 [2026-11-15 02:55:42,726] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf +rank0 [2026-11-15 02:55:43,152] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:44,813] INFO : Plotted jets +rank0 [2026-11-15 02:55:45,843] INFO : Plotted jet ratio +rank0 [2026-11-15 02:55:45,854] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:55:47,365] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:55:47,805] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:49,123] INFO : Plotted jets +rank0 [2026-11-15 02:55:50,137] INFO : Plotted jet ratio +rank0 [2026-11-15 02:55:50,147] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:55:51,538] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:55:51,979] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:53,278] INFO : Plotted jets +rank0 [2026-11-15 02:55:54,260] INFO : Plotted jet ratio +rank0 [2026-11-15 02:59:47,747] INFO : Step 9000/500000 rank0 | Train Loss: 2.3927 | Train Loss Unweighted: 1.0289 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2665 | Regression_cos_phi: 0.0063 | Regression_energy: 0.3630 | Regression_eta: 0.0328 | Regression_pt: 0.3473 | Regression_sin_phi: 0.0127 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1103 | Classification_binary: 0.2665 | Regression_cos_phi: 0.3672 | Regression_energy: 0.2172 | Regression_eta: 0.4262 | Regression_pt: 0.2077 | Regression_sin_phi: 0.7976 | LR: 2.43e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.5016s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0699 | max_loss: 0.0027 | max_backward: 0.1544 | max_optimizer_step: 0.0643 | max_tensorboard_logging: 0.9426 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5950.2 | uss_MB: 4729.3 | child_rss_MB: 23833.7 | allocated_MB: 198.3 | reserved_MB: 19218.0 | inactive_split_MB: 385.7 +rank0 [2026-11-15 02:59:47,751] INFO : Running validation on rank0 +rank0 [2026-11-15 02:59:47,751] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:59:47,752] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:59:48,382] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 02:59:48,506] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:59:48,619] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:59:48,691] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:59:48,759] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:59:48,826] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:59:48,894] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:59:48,962] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:59:49,067] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:59:49,173] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:59:49,277] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:59:49,345] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:59:49,414] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:59:49,483] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:59:49,550] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:59:49,617] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:59:49,721] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:59:49,741] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:59:49,753] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:59:49,756] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:59:49,766] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:59:49,779] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:59:49,782] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:59:49,792] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:59:49,805] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:59:49,808] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:59:49,819] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:59:49,822] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:59:49,825] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:59:49,827] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:59:49,830] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:59:49,832] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:59:57,576] INFO : VALIDATION | Step=9000/500000 | Train Loss=2.3927 | Valid Loss=3.1234 | Train Loss Unweighted=1.0289 | Valid Loss Unweighted=1.0859 | Stale=0 | ETA=2237.9m +rank0 [2026-11-15 02:59:57,576] INFO : split_configs=['1'] +rank0 [2026-11-15 02:59:57,643] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:59:57,646] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:59:57,719] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:59:58,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:59:58,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:59:59,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:59:59,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:59:59,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:00:00,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:00:00,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:00:00,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:00:01,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:00:01,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:00:01,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:00:02,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:00:02,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:00:03,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:00:03,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:00:03,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:00:03,815] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:00:03,819] INFO : split_configs=['1'] +rank0 [2026-11-15 03:00:03,863] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:00:03,866] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:00:03,907] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:00:04,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:00:05,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:00:05,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:00:06,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:00:06,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:00:06,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:00:07,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:00:07,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:00:08,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:00:08,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:00:09,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:00:09,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:00:10,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:00:10,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:00:10,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:00:11,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:00:11,456] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:00:11,460] INFO : split_configs=['1'] +rank0 [2026-11-15 03:00:11,470] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:00:11,475] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:00:11,517] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:00:12,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:00:12,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:00:13,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:00:13,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:00:14,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:00:14,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:00:14,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:00:15,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:00:15,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:00:16,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:00:16,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:00:16,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:00:17,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:00:17,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:00:18,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:00:18,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:00:18,536] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:00:18,546] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf +rank0 [2026-11-15 03:00:19,657] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf +rank0 [2026-11-15 03:00:20,088] INFO : Plotted number of elements +rank0 [2026-11-15 03:00:21,761] INFO : Plotted jets +rank0 [2026-11-15 03:00:22,777] INFO : Plotted jet ratio +rank0 [2026-11-15 03:00:22,789] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:00:24,356] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:00:24,792] INFO : Plotted number of elements +rank0 [2026-11-15 03:00:26,131] INFO : Plotted jets +rank0 [2026-11-15 03:00:27,152] INFO : Plotted jet ratio +rank0 [2026-11-15 03:00:27,160] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:00:28,582] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:00:29,024] INFO : Plotted number of elements +rank0 [2026-11-15 03:00:30,335] INFO : Plotted jets +rank0 [2026-11-15 03:00:31,319] INFO : Plotted jet ratio +rank0 [2026-11-15 03:00:31,712] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:00:31,712] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:04:26,047] INFO : Step 10000/500000 rank0 | Train Loss: 3.9013 | Train Loss Unweighted: 1.2087 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2662 | Regression_cos_phi: 0.0134 | Regression_energy: 0.4160 | Regression_eta: 0.1035 | Regression_pt: 0.3949 | Regression_sin_phi: 0.0141 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1303 | Classification_binary: 0.2662 | Regression_cos_phi: 0.7880 | Regression_energy: 0.2490 | Regression_eta: 1.3442 | Regression_pt: 0.2362 | Regression_sin_phi: 0.8874 | LR: 2.52e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.7200s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0651 | max_loss: 0.0028 | max_backward: 0.1420 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 1.1915 | max_tensorboard_flush: 0.0008 | Memory: rss_MB: 6110.1 | uss_MB: 4889.0 | child_rss_MB: 23751.4 | allocated_MB: 196.4 | reserved_MB: 18374.0 | inactive_split_MB: 289.6 +rank0 [2026-11-15 03:04:26,049] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 03:04:26,115] INFO : Running validation on rank0 +rank0 [2026-11-15 03:04:26,115] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:04:26,115] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:04:26,736] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 03:04:26,854] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:04:26,968] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:04:27,037] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:04:27,104] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:04:27,169] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:04:27,236] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:04:27,302] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:04:27,405] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:04:27,507] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:04:27,608] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:04:27,675] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:04:27,741] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:04:27,806] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:04:27,872] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:04:27,939] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:04:28,042] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:04:28,061] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:04:28,070] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:04:28,073] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:04:28,083] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:04:28,096] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:04:28,099] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:04:28,102] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:04:28,105] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:04:28,107] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:04:28,110] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:04:28,112] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:04:28,115] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:04:28,117] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:04:28,120] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:04:28,122] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:04:35,856] INFO : VALIDATION | Step=10000/500000 | Train Loss=3.9013 | Valid Loss=3.1129 | Train Loss Unweighted=1.2087 | Valid Loss Unweighted=1.0716 | Stale=0 | ETA=2237.2m +rank0 [2026-11-15 03:04:35,857] INFO : split_configs=['1'] +rank0 [2026-11-15 03:04:35,906] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:04:35,911] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:04:35,992] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:04:36,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:04:37,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:04:37,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:04:37,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:04:38,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:04:38,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:04:38,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:04:39,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:04:39,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:04:39,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:04:40,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:04:40,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:04:40,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:04:41,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:04:41,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:04:41,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:04:41,940] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:04:41,943] INFO : split_configs=['1'] +rank0 [2026-11-15 03:04:41,985] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:04:41,988] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:04:42,035] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:04:42,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:04:43,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:04:43,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:04:44,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:04:44,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:04:45,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:04:45,554] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:04:46,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:04:46,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:04:46,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:04:47,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:04:47,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:04:48,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:04:48,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:04:49,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:04:49,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:04:49,583] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:04:49,586] INFO : split_configs=['1'] +rank0 [2026-11-15 03:04:49,623] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:04:49,629] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:04:49,675] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:04:50,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:04:50,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:04:51,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:04:51,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:04:52,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:04:52,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:04:53,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:04:53,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:04:53,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:04:54,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:04:54,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:04:55,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:04:55,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:04:55,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:04:56,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:04:56,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:04:56,770] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:04:56,777] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf +rank0 [2026-11-15 03:04:57,868] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf +rank0 [2026-11-15 03:04:58,303] INFO : Plotted number of elements +rank0 [2026-11-15 03:04:59,977] INFO : Plotted jets +rank0 [2026-11-15 03:05:00,997] INFO : Plotted jet ratio +rank0 [2026-11-15 03:05:01,007] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:05:02,566] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:05:02,997] INFO : Plotted number of elements +rank0 [2026-11-15 03:05:04,325] INFO : Plotted jets +rank0 [2026-11-15 03:05:05,339] INFO : Plotted jet ratio +rank0 [2026-11-15 03:05:05,348] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:05:06,760] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:05:07,196] INFO : Plotted number of elements +rank0 [2026-11-15 03:05:08,499] INFO : Plotted jets +rank0 [2026-11-15 03:05:09,470] INFO : Plotted jet ratio +rank0 [2026-11-15 03:07:05,250] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 03:07:05,251] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:09:02,394] INFO : Step 11000/500000 rank0 | Train Loss: 2.6454 | Train Loss Unweighted: 0.9620 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2613 | Regression_cos_phi: 0.0130 | Regression_energy: 0.3284 | Regression_eta: 0.0398 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1394 | Classification_binary: 0.2613 | Regression_cos_phi: 0.7652 | Regression_energy: 0.1966 | Regression_eta: 0.5175 | Regression_pt: 0.1852 | Regression_sin_phi: 0.5803 | LR: 2.63e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.3790s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0651 | max_loss: 0.0028 | max_backward: 0.1412 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.8517 | max_tensorboard_flush: 0.0009 | Memory: rss_MB: 6158.4 | uss_MB: 4937.4 | child_rss_MB: 23908.5 | allocated_MB: 198.3 | reserved_MB: 17166.0 | inactive_split_MB: 369.7 +rank0 [2026-11-15 03:09:02,396] INFO : Running validation on rank0 +rank0 [2026-11-15 03:09:02,396] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:09:02,396] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:09:03,009] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 03:09:03,129] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:09:03,243] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:09:03,313] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:09:03,380] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:09:03,446] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:09:03,514] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:09:03,581] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:09:03,686] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:09:03,787] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:09:03,887] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:09:03,953] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:09:04,018] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:09:04,083] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:09:04,149] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:09:04,214] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:09:04,314] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:09:04,321] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:09:04,324] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:09:04,326] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:09:04,329] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:09:04,331] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:09:04,334] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:09:04,336] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:09:04,339] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:09:04,341] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:09:04,343] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:09:04,346] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:09:04,348] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:09:04,351] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:09:04,353] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:09:04,356] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:09:12,043] INFO : VALIDATION | Step=11000/500000 | Train Loss=2.6454 | Valid Loss=3.0495 | Train Loss Unweighted=0.9620 | Valid Loss Unweighted=1.0586 | Stale=0 | ETA=2234.3m +rank0 [2026-11-15 03:09:12,044] INFO : split_configs=['1'] +rank0 [2026-11-15 03:09:12,054] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:09:12,075] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:09:12,147] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:09:12,916] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:09:13,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:09:13,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:09:13,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:09:14,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:09:14,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:09:14,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:09:15,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:09:15,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:09:15,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:09:16,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:09:16,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:09:16,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:09:17,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:09:17,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:09:17,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:09:18,014] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:09:18,017] INFO : split_configs=['1'] +rank0 [2026-11-15 03:09:18,028] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:09:18,032] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:09:18,079] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:09:19,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:09:19,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:09:19,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:09:20,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:09:20,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:09:21,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:09:21,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:09:22,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:09:22,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:09:22,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:09:23,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:09:23,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:09:24,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:09:24,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:09:25,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:09:25,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:09:25,688] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:09:25,691] INFO : split_configs=['1'] +rank0 [2026-11-15 03:09:25,702] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:09:25,706] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:09:25,756] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:09:26,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:09:27,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:09:27,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:09:27,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:09:28,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:09:28,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:09:29,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:09:29,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:09:29,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:09:30,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:09:30,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:09:31,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:09:31,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:09:31,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:09:32,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:09:32,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:09:32,843] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:09:32,851] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf +rank0 [2026-11-15 03:09:33,947] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf +rank0 [2026-11-15 03:09:34,369] INFO : Plotted number of elements +rank0 [2026-11-15 03:09:36,072] INFO : Plotted jets +rank0 [2026-11-15 03:09:37,090] INFO : Plotted jet ratio +rank0 [2026-11-15 03:09:37,100] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:09:38,710] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:09:39,145] INFO : Plotted number of elements +rank0 [2026-11-15 03:09:40,463] INFO : Plotted jets +rank0 [2026-11-15 03:09:41,479] INFO : Plotted jet ratio +rank0 [2026-11-15 03:09:41,487] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:09:42,923] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:09:43,367] INFO : Plotted number of elements +rank0 [2026-11-15 03:09:44,666] INFO : Plotted jets +rank0 [2026-11-15 03:09:45,644] INFO : Plotted jet ratio +rank0 [2026-11-15 03:13:39,004] INFO : Step 12000/500000 rank0 | Train Loss: 2.7292 | Train Loss Unweighted: 1.0822 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2527 | Regression_cos_phi: 0.0121 | Regression_energy: 0.3935 | Regression_eta: 0.0387 | Regression_pt: 0.3739 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1186 | Classification_binary: 0.2527 | Regression_cos_phi: 0.7119 | Regression_energy: 0.2355 | Regression_eta: 0.5022 | Regression_pt: 0.2236 | Regression_sin_phi: 0.6847 | LR: 2.75e-05 | DataLoad Time: 0.0020s | Model Forward Time: 1.3928s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0659 | max_loss: 0.0027 | max_backward: 0.1404 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0010 | Memory: rss_MB: 4566.1 | uss_MB: 3345.1 | child_rss_MB: 23727.5 | allocated_MB: 196.8 | reserved_MB: 21524.0 | inactive_split_MB: 155.2 +rank0 [2026-11-15 03:13:39,005] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 03:13:39,073] INFO : Running validation on rank0 +rank0 [2026-11-15 03:13:39,074] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:13:39,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:13:39,698] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 03:13:39,823] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:13:39,935] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:13:40,005] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:13:40,075] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:13:40,143] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:13:40,210] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:13:40,278] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:13:40,384] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:13:40,486] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:13:40,588] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:13:40,654] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:13:40,719] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:13:40,784] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:13:40,849] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:13:40,915] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:13:41,016] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:13:41,023] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:13:41,026] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:13:41,028] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:13:41,031] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:13:41,034] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:13:41,036] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:13:41,039] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:13:41,041] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:13:41,044] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:13:41,046] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:13:41,049] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:13:41,051] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:13:41,054] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:13:41,056] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:13:41,059] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:13:48,797] INFO : VALIDATION | Step=12000/500000 | Train Loss=2.7292 | Valid Loss=3.0405 | Train Loss Unweighted=1.0822 | Valid Loss Unweighted=1.0477 | Stale=0 | ETA=2231.5m +rank0 [2026-11-15 03:13:48,797] INFO : split_configs=['1'] +rank0 [2026-11-15 03:13:48,808] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:13:48,811] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:13:48,886] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:13:49,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:13:49,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:13:50,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:13:50,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:13:51,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:13:51,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:13:51,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:13:52,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:13:52,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:13:52,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:13:53,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:13:53,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:13:53,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:13:54,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:13:54,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:13:54,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:13:54,743] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:13:54,746] INFO : split_configs=['1'] +rank0 [2026-11-15 03:13:54,757] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:13:54,760] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:13:54,802] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:13:55,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:13:56,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:13:56,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:13:57,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:13:57,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:13:57,890] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:13:58,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:13:58,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:13:59,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:13:59,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:14:00,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:14:00,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:14:01,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:14:01,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:14:01,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:14:02,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:14:02,374] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:14:02,380] INFO : split_configs=['1'] +rank0 [2026-11-15 03:14:02,392] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:14:02,395] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:14:02,438] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:14:03,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:14:03,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:14:04,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:14:04,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:14:04,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:14:05,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:14:05,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:14:06,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:14:06,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:14:07,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:14:07,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:14:07,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:14:08,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:14:08,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:14:09,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:14:09,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:14:09,550] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:14:09,557] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf +rank0 [2026-11-15 03:14:10,697] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf +rank0 [2026-11-15 03:14:11,137] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:12,835] INFO : Plotted jets +rank0 [2026-11-15 03:14:13,860] INFO : Plotted jet ratio +rank0 [2026-11-15 03:14:13,872] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:14:15,473] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:14:15,933] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:17,270] INFO : Plotted jets +rank0 [2026-11-15 03:14:18,292] INFO : Plotted jet ratio +rank0 [2026-11-15 03:14:18,300] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:14:19,717] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:14:20,175] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:21,481] INFO : Plotted jets +rank0 [2026-11-15 03:14:22,463] INFO : Plotted jet ratio +rank0 [2026-11-15 03:14:22,846] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 03:14:22,847] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:16,444] INFO : Step 13000/500000 rank0 | Train Loss: 3.1323 | Train Loss Unweighted: 1.2570 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2596 | Regression_cos_phi: 0.0180 | Regression_energy: 0.4838 | Regression_eta: 0.0260 | Regression_pt: 0.4567 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1397 | Classification_binary: 0.2596 | Regression_cos_phi: 1.0580 | Regression_energy: 0.2896 | Regression_eta: 0.3372 | Regression_pt: 0.2732 | Regression_sin_phi: 0.7750 | LR: 2.88e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.4011s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0652 | max_loss: 0.0030 | max_backward: 0.1419 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.8528 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 6324.3 | uss_MB: 5103.2 | child_rss_MB: 23860.7 | allocated_MB: 198.1 | reserved_MB: 18360.0 | inactive_split_MB: 177.9 +rank0 [2026-11-15 03:18:16,448] INFO : Running validation on rank0 +rank0 [2026-11-15 03:18:16,448] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:18:16,448] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:18:17,071] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 03:18:17,191] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:18:17,304] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:18:17,372] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:18:17,438] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:18:17,502] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:18:17,567] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:18:17,633] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:18:17,739] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:18:17,845] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:18:17,950] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:18:18,018] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:18:18,087] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:18:18,154] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:18:18,220] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:18:18,285] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:18:18,386] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:18:18,393] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:18:18,396] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:18:18,399] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:18:18,402] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:18:18,404] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:18:18,407] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:18:18,409] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:18:18,412] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:18:18,414] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:18:18,416] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:18:18,419] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:18:18,421] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:18:18,424] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:18:18,426] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:18:18,429] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:18:26,146] INFO : VALIDATION | Step=13000/500000 | Train Loss=3.1323 | Valid Loss=3.0005 | Train Loss Unweighted=1.2570 | Valid Loss Unweighted=1.0367 | Stale=0 | ETA=2228.8m +rank0 [2026-11-15 03:18:26,146] INFO : split_configs=['1'] +rank0 [2026-11-15 03:18:26,158] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:18:26,161] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:18:26,241] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:27,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:27,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:27,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:28,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:28,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:28,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:29,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:29,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:29,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:30,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:30,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:30,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:31,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:31,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:31,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:32,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:32,241] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:18:32,244] INFO : split_configs=['1'] +rank0 [2026-11-15 03:18:32,255] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:18:32,263] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:18:32,313] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:33,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:33,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:34,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:34,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:35,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:35,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:35,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:36,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:36,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:37,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:37,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:38,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:38,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:39,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:39,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:39,916] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:39,987] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:18:39,991] INFO : split_configs=['1'] +rank0 [2026-11-15 03:18:40,002] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:18:40,005] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:18:40,057] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:40,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:41,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:41,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:42,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:42,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:43,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:43,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:43,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:44,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:44,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:45,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:45,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:45,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:46,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:46,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:47,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:47,219] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:18:47,226] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf +rank0 [2026-11-15 03:18:48,341] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf +rank0 [2026-11-15 03:18:48,768] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:50,484] INFO : Plotted jets +rank0 [2026-11-15 03:18:51,516] INFO : Plotted jet ratio +rank0 [2026-11-15 03:18:51,529] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:18:53,089] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:18:53,527] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:54,855] INFO : Plotted jets +rank0 [2026-11-15 03:18:55,883] INFO : Plotted jet ratio +rank0 [2026-11-15 03:18:55,893] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:18:57,285] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:18:57,736] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:59,052] INFO : Plotted jets +rank0 [2026-11-15 03:19:00,045] INFO : Plotted jet ratio +rank0 [2026-11-15 03:20:56,366] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 03:20:56,366] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:53,704] INFO : Step 14000/500000 rank0 | Train Loss: 2.9143 | Train Loss Unweighted: 1.0571 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2785 | Regression_cos_phi: 0.0120 | Regression_energy: 0.3733 | Regression_eta: 0.0279 | Regression_pt: 0.3493 | Regression_sin_phi: 0.0155 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1575 | Classification_binary: 0.2785 | Regression_cos_phi: 0.7069 | Regression_energy: 0.2234 | Regression_eta: 0.3623 | Regression_pt: 0.2089 | Regression_sin_phi: 0.9768 | LR: 3.02e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.3310s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0591 | max_loss: 0.0030 | max_backward: 0.1265 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0009 | Memory: rss_MB: 6405.4 | uss_MB: 5196.3 | child_rss_MB: 23666.8 | allocated_MB: 196.6 | reserved_MB: 18680.0 | inactive_split_MB: 177.4 +rank0 [2026-11-15 03:22:53,709] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 03:22:53,774] INFO : Running validation on rank0 +rank0 [2026-11-15 03:22:53,774] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:22:53,774] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:22:54,391] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 03:22:54,508] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:22:54,618] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:22:54,686] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:22:54,753] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:22:54,818] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:22:54,885] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:22:54,952] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:22:55,059] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:22:55,165] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:22:55,268] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:22:55,333] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:22:55,398] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:22:55,462] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:22:55,528] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:22:55,592] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:22:55,692] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:22:55,699] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:22:55,702] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:22:55,704] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:22:55,707] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:22:55,710] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:22:55,712] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:22:55,714] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:22:55,717] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:22:55,719] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:22:55,721] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:22:55,724] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:22:55,726] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:22:55,729] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:22:55,731] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:22:55,734] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:23:03,435] INFO : VALIDATION | Step=14000/500000 | Train Loss=2.9143 | Valid Loss=2.9835 | Train Loss Unweighted=1.0571 | Valid Loss Unweighted=1.0255 | Stale=0 | ETA=2225.8m +rank0 [2026-11-15 03:23:03,435] INFO : split_configs=['1'] +rank0 [2026-11-15 03:23:03,446] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:23:03,454] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:23:03,546] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:23:04,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:23:04,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:23:05,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:23:05,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:23:05,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:23:06,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:23:06,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:23:06,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:23:07,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:23:07,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:23:07,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:23:08,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:23:08,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:23:08,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:23:09,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:23:09,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:23:09,441] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:23:09,443] INFO : split_configs=['1'] +rank0 [2026-11-15 03:23:09,455] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:23:09,458] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:23:09,511] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:23:10,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:23:10,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:23:11,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:23:11,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:23:12,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:23:12,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:23:13,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:23:13,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:23:14,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:23:14,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:23:14,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:23:15,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:23:15,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:23:16,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:23:16,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:23:17,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:23:17,156] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:23:17,160] INFO : split_configs=['1'] +rank0 [2026-11-15 03:23:17,171] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:23:17,186] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:23:17,236] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:23:18,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:23:18,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:23:18,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:23:19,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:23:19,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:23:20,217] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:23:20,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:23:21,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:23:21,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:23:21,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:23:22,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:23:22,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:23:23,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:23:23,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:23:23,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:23:24,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:23:24,329] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:23:24,337] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf +rank0 [2026-11-15 03:23:25,442] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf +rank0 [2026-11-15 03:23:25,867] INFO : Plotted number of elements +rank0 [2026-11-15 03:23:27,548] INFO : Plotted jets +rank0 [2026-11-15 03:23:28,575] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:28,586] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:23:30,140] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:23:30,584] INFO : Plotted number of elements +rank0 [2026-11-15 03:23:31,899] INFO : Plotted jets +rank0 [2026-11-15 03:23:32,919] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:32,928] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:23:34,315] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:23:34,757] INFO : Plotted number of elements +rank0 [2026-11-15 03:23:36,065] INFO : Plotted jets +rank0 [2026-11-15 03:23:37,046] INFO : Plotted jet ratio +rank0 [2026-11-15 03:27:29,902] INFO : Step 15000/500000 rank0 | Train Loss: 2.4760 | Train Loss Unweighted: 0.9661 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2485 | Regression_cos_phi: 0.0127 | Regression_energy: 0.3365 | Regression_eta: 0.0343 | Regression_pt: 0.3251 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0850 | Classification_binary: 0.2485 | Regression_cos_phi: 0.7448 | Regression_energy: 0.2014 | Regression_eta: 0.4450 | Regression_pt: 0.1944 | Regression_sin_phi: 0.5569 | LR: 3.17e-05 | DataLoad Time: 0.0024s | Model Forward Time: 1.5519s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0024 | max_forward: 0.0557 | max_loss: 0.0028 | max_backward: 0.1190 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 1.0551 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 4875.8 | uss_MB: 3666.6 | child_rss_MB: 23529.2 | allocated_MB: 194.5 | reserved_MB: 17698.0 | inactive_split_MB: 171.5 +rank0 [2026-11-15 03:27:29,905] INFO : Running validation on rank0 +rank0 [2026-11-15 03:27:29,905] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:27:29,905] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:27:30,528] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 03:27:30,651] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:27:30,763] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:27:30,832] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:27:30,897] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:27:30,962] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:27:31,029] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:27:31,099] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:27:31,206] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:27:31,312] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:27:31,415] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:27:31,482] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:27:31,548] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:27:31,612] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:27:31,677] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:27:31,743] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:27:31,843] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:27:31,850] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:27:31,853] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:27:31,855] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:27:31,858] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:27:31,860] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:27:31,863] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:27:31,865] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:27:31,868] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:27:31,870] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:27:31,872] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:27:31,875] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:27:31,877] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:27:31,880] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:27:31,882] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:27:31,885] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:27:39,602] INFO : VALIDATION | Step=15000/500000 | Train Loss=2.4760 | Valid Loss=2.9111 | Train Loss Unweighted=0.9661 | Valid Loss Unweighted=1.0166 | Stale=0 | ETA=2222.0m +rank0 [2026-11-15 03:27:39,603] INFO : split_configs=['1'] +rank0 [2026-11-15 03:27:39,647] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:27:39,650] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:27:39,723] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:27:40,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:27:40,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:27:41,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:27:41,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:27:41,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:27:42,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:27:42,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:27:42,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:27:43,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:27:43,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:27:43,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:27:44,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:27:44,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:27:44,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:27:45,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:27:45,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:27:45,576] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:27:45,580] INFO : split_configs=['1'] +rank0 [2026-11-15 03:27:45,639] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:27:45,642] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:27:45,689] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:27:46,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:27:47,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:27:47,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:27:47,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:27:48,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:27:48,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:27:49,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:27:49,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:27:50,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:27:50,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:27:51,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:27:51,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:27:51,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:27:52,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:27:52,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:27:53,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:27:53,335] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:27:53,338] INFO : split_configs=['1'] +rank0 [2026-11-15 03:27:53,378] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:27:53,381] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:27:53,429] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:27:54,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:27:54,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:27:55,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:27:55,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:27:55,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:27:56,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:27:56,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:27:57,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:27:57,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:27:57,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:27:58,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:27:58,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:27:59,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:27:59,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:28:00,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:28:00,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:28:00,505] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:28:00,523] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf +rank0 [2026-11-15 03:28:01,648] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf +rank0 [2026-11-15 03:28:02,088] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:03,800] INFO : Plotted jets +rank0 [2026-11-15 03:28:04,867] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:04,879] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:28:06,478] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:28:06,921] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:08,288] INFO : Plotted jets +rank0 [2026-11-15 03:28:09,307] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:09,316] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:28:10,774] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:28:11,216] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:12,536] INFO : Plotted jets +rank0 [2026-11-15 03:28:13,521] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:13,919] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 03:28:13,919] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:08,677] INFO : Step 16000/500000 rank0 | Train Loss: 2.4377 | Train Loss Unweighted: 1.0890 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2703 | Regression_cos_phi: 0.0096 | Regression_energy: 0.3958 | Regression_eta: 0.0417 | Regression_pt: 0.3632 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1073 | Classification_binary: 0.2703 | Regression_cos_phi: 0.5633 | Regression_energy: 0.2368 | Regression_eta: 0.5419 | Regression_pt: 0.2172 | Regression_sin_phi: 0.5009 | LR: 3.33e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.7058s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0617 | max_loss: 0.0030 | max_backward: 0.1307 | max_optimizer_step: 0.0703 | max_tensorboard_logging: 2.1717 | max_tensorboard_flush: 0.0012 | Memory: rss_MB: 6363.9 | uss_MB: 5218.2 | child_rss_MB: 23765.8 | allocated_MB: 199.0 | reserved_MB: 20632.0 | inactive_split_MB: 167.0 +rank0 [2026-11-15 03:32:08,680] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 03:32:08,744] INFO : Running validation on rank0 +rank0 [2026-11-15 03:32:08,744] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:32:08,744] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:32:09,368] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 03:32:09,487] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:32:09,603] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:32:09,673] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:32:09,740] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:32:09,806] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:32:09,872] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:32:09,938] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:32:10,050] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:32:10,155] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:32:10,257] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:32:10,325] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:32:10,391] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:32:10,456] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:32:10,521] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:32:10,586] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:32:10,688] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:32:10,710] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:32:10,722] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:32:10,725] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:32:10,728] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:32:10,731] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:32:10,733] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:32:10,736] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:32:10,752] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:32:10,756] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:32:10,771] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:32:10,773] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:32:10,776] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:32:10,778] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:32:10,781] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:32:10,783] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:32:18,510] INFO : VALIDATION | Step=16000/500000 | Train Loss=2.4377 | Valid Loss=2.9038 | Train Loss Unweighted=1.0890 | Valid Loss Unweighted=1.0098 | Stale=0 | ETA=2219.4m +rank0 [2026-11-15 03:32:18,510] INFO : split_configs=['1'] +rank0 [2026-11-15 03:32:18,560] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:32:18,568] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:32:18,650] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:19,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:19,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:20,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:20,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:20,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:21,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:21,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:21,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:22,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:22,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:22,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:23,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:23,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:23,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:24,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:24,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:24,620] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:32:24,623] INFO : split_configs=['1'] +rank0 [2026-11-15 03:32:24,712] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:32:24,728] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:32:24,779] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:25,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:26,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:26,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:27,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:27,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:27,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:28,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:28,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:29,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:29,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:30,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:30,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:31,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:31,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:31,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:32,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:32,449] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:32:32,453] INFO : split_configs=['1'] +rank0 [2026-11-15 03:32:32,557] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:32:32,561] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:32:32,616] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:33,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:33,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:34,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:34,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:35,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:35,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:36,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:36,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:36,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:37,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:37,656] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:38,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:38,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:38,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:39,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:39,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:39,818] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:32:39,837] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf +rank0 [2026-11-15 03:32:40,947] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf +rank0 [2026-11-15 03:32:41,384] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:43,079] INFO : Plotted jets +rank0 [2026-11-15 03:32:44,108] INFO : Plotted jet ratio +rank0 [2026-11-15 03:32:44,122] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:32:45,709] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:32:46,165] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:47,494] INFO : Plotted jets +rank0 [2026-11-15 03:32:48,523] INFO : Plotted jet ratio +rank0 [2026-11-15 03:32:48,533] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:32:49,955] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:32:50,407] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:51,722] INFO : Plotted jets +rank0 [2026-11-15 03:32:52,714] INFO : Plotted jet ratio +rank0 [2026-11-15 03:34:48,766] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 03:34:48,766] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:36:46,317] INFO : Step 17000/500000 rank0 | Train Loss: 2.0359 | Train Loss Unweighted: 0.8891 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2597 | Regression_cos_phi: 0.0061 | Regression_energy: 0.2984 | Regression_eta: 0.0271 | Regression_pt: 0.2874 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0868 | Classification_binary: 0.2597 | Regression_cos_phi: 0.3558 | Regression_energy: 0.1786 | Regression_eta: 0.3519 | Regression_pt: 0.1719 | Regression_sin_phi: 0.6312 | LR: 3.51e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.5780s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0663 | max_loss: 0.0029 | max_backward: 0.1438 | max_optimizer_step: 0.0712 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6632.7 | uss_MB: 5423.6 | child_rss_MB: 23737.1 | allocated_MB: 196.0 | reserved_MB: 19958.0 | inactive_split_MB: 146.0 +rank0 [2026-11-15 03:36:46,320] INFO : Running validation on rank0 +rank0 [2026-11-15 03:36:46,320] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:36:46,320] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:36:46,935] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 03:36:47,054] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:36:47,168] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:36:47,235] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:36:47,300] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:36:47,365] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:36:47,431] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:36:47,497] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:36:47,602] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:36:47,706] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:36:47,810] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:36:47,878] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:36:47,945] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:36:48,011] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:36:48,079] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:36:48,144] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:36:48,244] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:36:48,250] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:36:48,251] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:36:48,252] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:36:48,253] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:36:48,255] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:36:48,256] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:36:48,257] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:36:48,258] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:36:48,259] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:36:48,277] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:36:48,278] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:36:48,279] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:36:48,280] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:36:48,281] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:36:48,282] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:36:56,030] INFO : VALIDATION | Step=17000/500000 | Train Loss=2.0359 | Valid Loss=2.8832 | Train Loss Unweighted=0.8891 | Valid Loss Unweighted=0.9973 | Stale=0 | ETA=2215.9m +rank0 [2026-11-15 03:36:56,030] INFO : split_configs=['1'] +rank0 [2026-11-15 03:36:56,098] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:36:56,103] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:36:56,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:36:56,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:36:57,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:36:57,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:36:58,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:36:58,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:36:58,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:36:59,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:36:59,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:36:59,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:37:00,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:00,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:00,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:01,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:01,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:01,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:02,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:02,101] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:37:02,105] INFO : split_configs=['1'] +rank0 [2026-11-15 03:37:02,167] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:37:02,171] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:37:02,220] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:37:03,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:37:03,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:37:04,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:37:04,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:37:04,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:37:05,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:37:05,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:37:06,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:37:06,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:37:07,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:07,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:08,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:08,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:08,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:09,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:09,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:09,868] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:37:09,872] INFO : split_configs=['1'] +rank0 [2026-11-15 03:37:09,934] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:37:09,937] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:37:09,985] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:37:10,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:37:11,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:37:11,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:37:12,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:37:12,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:37:12,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:37:13,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:37:13,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:37:14,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:37:14,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:14,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:15,382] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:15,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:16,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:16,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:17,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:17,103] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:37:17,114] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf +rank0 [2026-11-15 03:37:18,241] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf +rank0 [2026-11-15 03:37:18,670] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:20,387] INFO : Plotted jets +rank0 [2026-11-15 03:37:21,407] INFO : Plotted jet ratio +rank0 [2026-11-15 03:37:21,424] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:37:23,042] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:37:23,479] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:24,811] INFO : Plotted jets +rank0 [2026-11-15 03:37:25,855] INFO : Plotted jet ratio +rank0 [2026-11-15 03:37:25,865] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:37:27,296] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:37:27,741] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:29,045] INFO : Plotted jets +rank0 [2026-11-15 03:37:30,028] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:22,927] INFO : Step 18000/500000 rank0 | Train Loss: 2.3095 | Train Loss Unweighted: 1.0294 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2439 | Regression_cos_phi: 0.0072 | Regression_energy: 0.3711 | Regression_eta: 0.0273 | Regression_pt: 0.3678 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1109 | Classification_binary: 0.2439 | Regression_cos_phi: 0.4242 | Regression_energy: 0.2221 | Regression_eta: 0.3551 | Regression_pt: 0.2200 | Regression_sin_phi: 0.7333 | LR: 3.69e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.3672s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0600 | max_loss: 0.0027 | max_backward: 0.1286 | max_optimizer_step: 0.0700 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6604.0 | uss_MB: 5394.9 | child_rss_MB: 23820.7 | allocated_MB: 198.4 | reserved_MB: 18404.0 | inactive_split_MB: 183.6 +rank0 [2026-11-15 03:41:22,929] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 03:41:22,998] INFO : Running validation on rank0 +rank0 [2026-11-15 03:41:22,998] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:41:22,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:41:23,614] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 03:41:23,731] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:41:23,843] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:41:23,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:41:23,978] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:41:24,043] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:41:24,112] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:41:24,180] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:41:24,288] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:41:24,393] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:41:24,495] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:41:24,560] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:41:24,626] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:41:24,691] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:41:24,756] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:41:24,821] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:41:24,921] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:41:24,928] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:41:24,931] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:41:24,933] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:41:24,935] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:41:24,938] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:41:24,940] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:41:24,943] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:41:24,945] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:41:24,948] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:41:24,950] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:41:24,953] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:41:24,955] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:41:24,957] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:41:24,960] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:41:24,962] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:41:32,600] INFO : VALIDATION | Step=18000/500000 | Train Loss=2.3095 | Valid Loss=2.9102 | Train Loss Unweighted=1.0294 | Valid Loss Unweighted=0.9902 | Stale=1 | ETA=2211.9m +rank0 [2026-11-15 03:41:32,601] INFO : split_configs=['1'] +rank0 [2026-11-15 03:41:32,612] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:41:32,616] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:41:32,703] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:33,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:33,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:34,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:34,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:34,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:35,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:35,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:35,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:36,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:36,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:36,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:37,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:37,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:37,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:38,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:38,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:38,658] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:41:38,661] INFO : split_configs=['1'] +rank0 [2026-11-15 03:41:38,698] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:41:38,701] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:41:38,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:39,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:40,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:40,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:41,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:41,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:41,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:42,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:42,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:43,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:43,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:44,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:44,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:45,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:45,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:45,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:46,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:46,483] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:41:46,486] INFO : split_configs=['1'] +rank0 [2026-11-15 03:41:46,528] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:41:46,535] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:41:46,590] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:47,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:47,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:48,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:48,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:49,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:49,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:50,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:50,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:50,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:51,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:51,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:52,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:52,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:52,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:53,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:53,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:53,749] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:41:53,756] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf +rank0 [2026-11-15 03:41:54,878] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf +rank0 [2026-11-15 03:41:55,304] INFO : Plotted number of elements +rank0 [2026-11-15 03:41:57,012] INFO : Plotted jets +rank0 [2026-11-15 03:41:58,064] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:58,078] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:41:59,645] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:42:00,076] INFO : Plotted number of elements +rank0 [2026-11-15 03:42:01,412] INFO : Plotted jets +rank0 [2026-11-15 03:42:02,431] INFO : Plotted jet ratio +rank0 [2026-11-15 03:42:02,440] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:42:03,881] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:42:04,313] INFO : Plotted number of elements +rank0 [2026-11-15 03:42:05,621] INFO : Plotted jets +rank0 [2026-11-15 03:42:06,597] INFO : Plotted jet ratio +rank0 [2026-11-15 03:42:06,990] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 03:42:06,990] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:45:59,875] INFO : Step 19000/500000 rank0 | Train Loss: 3.6684 | Train Loss Unweighted: 1.0602 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2522 | Regression_cos_phi: 0.0094 | Regression_energy: 0.3671 | Regression_eta: 0.0825 | Regression_pt: 0.3286 | Regression_sin_phi: 0.0199 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1244 | Classification_binary: 0.2522 | Regression_cos_phi: 0.5523 | Regression_energy: 0.2197 | Regression_eta: 1.0717 | Regression_pt: 0.1965 | Regression_sin_phi: 1.2515 | LR: 3.88e-05 | DataLoad Time: 0.0023s | Model Forward Time: 1.4804s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0023 | max_forward: 0.0550 | max_loss: 0.0029 | max_backward: 0.1179 | max_optimizer_step: 0.0718 | max_tensorboard_logging: 0.9567 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 6745.0 | uss_MB: 5535.9 | child_rss_MB: 23734.8 | allocated_MB: 195.1 | reserved_MB: 19920.0 | inactive_split_MB: 278.9 +rank0 [2026-11-15 03:45:59,878] INFO : Running validation on rank0 +rank0 [2026-11-15 03:45:59,879] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:45:59,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:46:00,495] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 03:46:00,616] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:46:00,733] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:46:00,803] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:46:00,870] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:46:00,935] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:46:01,001] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:46:01,069] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:46:01,175] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:46:01,276] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:46:01,377] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:46:01,444] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:46:01,510] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:46:01,574] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:46:01,640] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:46:01,704] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:46:01,804] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:46:01,811] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:46:01,814] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:46:01,816] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:46:01,819] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:46:01,822] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:46:01,824] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:46:01,826] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:46:01,829] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:46:01,831] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:46:01,833] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:46:01,836] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:46:01,838] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:46:01,841] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:46:01,843] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:46:01,846] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:46:09,638] INFO : VALIDATION | Step=19000/500000 | Train Loss=3.6684 | Valid Loss=2.8504 | Train Loss Unweighted=1.0602 | Valid Loss Unweighted=0.9862 | Stale=0 | ETA=2208.1m +rank0 [2026-11-15 03:46:09,638] INFO : split_configs=['1'] +rank0 [2026-11-15 03:46:09,682] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:46:09,689] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:46:09,804] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:10,650] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:11,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:11,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:11,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:12,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:12,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:12,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:13,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:13,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:13,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:14,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:14,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:14,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:15,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:15,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:15,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:15,766] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:46:15,769] INFO : split_configs=['1'] +rank0 [2026-11-15 03:46:15,780] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:46:15,785] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:46:15,839] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:16,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:17,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:17,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:18,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:18,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:19,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:19,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:19,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:20,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:20,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:21,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:21,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:22,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:22,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:23,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:23,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:23,595] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:46:23,600] INFO : split_configs=['1'] +rank0 [2026-11-15 03:46:23,611] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:46:23,615] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:46:23,671] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:24,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:25,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:25,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:25,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:26,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:26,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:27,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:27,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:27,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:28,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:28,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:29,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:29,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:30,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:30,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:30,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:30,915] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:46:30,922] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf +rank0 [2026-11-15 03:46:32,042] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf +rank0 [2026-11-15 03:46:32,467] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:33,751] INFO : Plotted jets +rank0 [2026-11-15 03:46:35,211] INFO : Plotted jet ratio +rank0 [2026-11-15 03:46:35,221] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:46:36,800] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:46:37,235] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:38,567] INFO : Plotted jets +rank0 [2026-11-15 03:46:39,583] INFO : Plotted jet ratio +rank0 [2026-11-15 03:46:39,593] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:46:40,989] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:46:41,442] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:42,739] INFO : Plotted jets +rank0 [2026-11-15 03:46:43,721] INFO : Plotted jet ratio +rank0 [2026-11-15 03:48:40,112] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 03:48:40,112] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:37,524] INFO : Step 20000/500000 rank0 | Train Loss: 2.1480 | Train Loss Unweighted: 0.8316 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2504 | Regression_cos_phi: 0.0039 | Regression_energy: 0.2645 | Regression_eta: 0.0437 | Regression_pt: 0.2582 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1302 | Classification_binary: 0.2504 | Regression_cos_phi: 0.2268 | Regression_energy: 0.1583 | Regression_eta: 0.5675 | Regression_pt: 0.1544 | Regression_sin_phi: 0.6603 | LR: 4.07e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.4362s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0708 | max_loss: 0.0029 | max_backward: 0.1559 | max_optimizer_step: 0.0730 | max_tensorboard_logging: 0.8509 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 6865.8 | uss_MB: 5657.0 | child_rss_MB: 23594.8 | allocated_MB: 198.1 | reserved_MB: 20462.0 | inactive_split_MB: 281.9 +rank0 [2026-11-15 03:50:37,527] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 03:50:37,595] INFO : Running validation on rank0 +rank0 [2026-11-15 03:50:37,596] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:50:37,596] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:50:38,213] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 03:50:38,332] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:50:38,444] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:50:38,513] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:50:38,578] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:50:38,642] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:50:38,707] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:50:38,809] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:50:38,923] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:50:39,027] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:50:39,129] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:50:39,198] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:50:39,264] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:50:39,330] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:50:39,397] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:50:39,464] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:50:39,568] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:50:39,588] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:50:39,600] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:50:39,603] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:50:39,613] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:50:39,626] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:50:39,629] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:50:39,639] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:50:39,652] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:50:39,655] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:50:39,658] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:50:39,673] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:50:39,675] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:50:39,676] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:50:39,677] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:50:39,678] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:50:47,413] INFO : VALIDATION | Step=20000/500000 | Train Loss=2.1480 | Valid Loss=2.8273 | Train Loss Unweighted=0.8316 | Valid Loss Unweighted=0.9721 | Stale=0 | ETA=2204.4m +rank0 [2026-11-15 03:50:47,414] INFO : split_configs=['1'] +rank0 [2026-11-15 03:50:47,425] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:50:47,428] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:50:47,507] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:50:48,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:50:48,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:50:49,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:50:49,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:50:49,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:50:50,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:50:50,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:50:50,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:50:51,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:50:51,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:50:51,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:50:52,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:50:52,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:50:52,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:50:53,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:50:53,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:50:53,502] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:50:53,506] INFO : split_configs=['1'] +rank0 [2026-11-15 03:50:53,518] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:50:53,521] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:50:53,571] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:50:54,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:50:54,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:50:55,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:50:55,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:50:56,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:50:56,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:50:57,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:50:57,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:50:58,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:50:58,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:50:58,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:50:59,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:50:59,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:51:00,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:51:00,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:51:01,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:51:01,186] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:51:01,189] INFO : split_configs=['1'] +rank0 [2026-11-15 03:51:01,200] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:51:01,203] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:51:01,254] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:51:02,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:51:02,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:51:03,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:51:03,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:51:03,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:51:04,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:51:04,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:51:05,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:51:05,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:51:05,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:51:06,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:51:06,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:51:07,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:51:07,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:51:07,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:51:08,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:51:08,374] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:51:08,382] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf +rank0 [2026-11-15 03:51:09,498] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf +rank0 [2026-11-15 03:51:09,926] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:11,631] INFO : Plotted jets +rank0 [2026-11-15 03:51:12,648] INFO : Plotted jet ratio +rank0 [2026-11-15 03:51:12,659] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:51:14,257] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:51:14,689] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:16,029] INFO : Plotted jets +rank0 [2026-11-15 03:51:17,047] INFO : Plotted jet ratio +rank0 [2026-11-15 03:51:17,057] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:51:18,459] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:51:18,901] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:20,209] INFO : Plotted jets +rank0 [2026-11-15 03:51:21,188] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:14,175] INFO : Step 21000/500000 rank0 | Train Loss: 3.8062 | Train Loss Unweighted: 1.0392 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2480 | Regression_cos_phi: 0.0162 | Regression_energy: 0.3656 | Regression_eta: 0.0863 | Regression_pt: 0.3074 | Regression_sin_phi: 0.0153 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1214 | Classification_binary: 0.2480 | Regression_cos_phi: 0.9509 | Regression_energy: 0.2188 | Regression_eta: 1.1212 | Regression_pt: 0.1838 | Regression_sin_phi: 0.9621 | LR: 4.28e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.3954s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0576 | max_loss: 0.0027 | max_backward: 0.1256 | max_optimizer_step: 0.0731 | max_tensorboard_logging: 0.8598 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6848.6 | uss_MB: 5639.8 | child_rss_MB: 23765.4 | allocated_MB: 195.7 | reserved_MB: 18816.0 | inactive_split_MB: 234.3 +rank0 [2026-11-15 03:55:14,179] INFO : Running validation on rank0 +rank0 [2026-11-15 03:55:14,179] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:55:14,179] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:55:14,804] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 03:55:14,924] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:55:15,040] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:55:15,111] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:55:15,177] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:55:15,242] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:55:15,308] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:55:15,372] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:55:15,480] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:55:15,582] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:55:15,683] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:55:15,750] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:55:15,815] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:55:15,878] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:55:15,943] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:55:16,008] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:55:16,111] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:55:16,134] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:55:16,147] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:55:16,150] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:55:16,160] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:55:16,173] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:55:16,176] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:55:16,177] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:55:16,178] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:55:16,179] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:55:16,180] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:55:16,181] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:55:16,182] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:55:16,183] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:55:16,194] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:55:16,195] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:55:23,936] INFO : VALIDATION | Step=21000/500000 | Train Loss=3.8062 | Valid Loss=2.8064 | Train Loss Unweighted=1.0392 | Valid Loss Unweighted=0.9680 | Stale=0 | ETA=2200.2m +rank0 [2026-11-15 03:55:23,936] INFO : split_configs=['1'] +rank0 [2026-11-15 03:55:23,947] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:55:23,962] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:55:24,055] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:55:24,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:55:25,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:55:25,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:55:25,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:55:26,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:55:26,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:55:26,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:55:27,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:55:27,639] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:55:27,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:55:28,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:55:28,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:55:28,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:55:29,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:55:29,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:55:29,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:55:30,034] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:55:30,037] INFO : split_configs=['1'] +rank0 [2026-11-15 03:55:30,048] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:55:30,054] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:55:30,112] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:55:31,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:55:31,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:55:32,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:55:32,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:55:32,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:55:33,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:55:33,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:55:34,226] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:55:34,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:55:35,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:55:35,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:55:36,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:55:36,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:55:36,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:55:37,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:55:37,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:55:37,854] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:55:37,857] INFO : split_configs=['1'] +rank0 [2026-11-15 03:55:37,868] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:55:37,872] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:55:37,931] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:55:38,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:55:39,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:55:39,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:55:40,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:55:40,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:55:41,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:55:41,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:55:41,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:55:42,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:55:42,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:55:43,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:55:43,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:55:43,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:55:44,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:55:44,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:55:45,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:55:45,161] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:55:45,170] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf +rank0 [2026-11-15 03:55:46,282] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf +rank0 [2026-11-15 03:55:46,716] INFO : Plotted number of elements +rank0 [2026-11-15 03:55:47,982] INFO : Plotted jets +rank0 [2026-11-15 03:55:49,430] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:49,443] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:55:50,998] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:55:51,449] INFO : Plotted number of elements +rank0 [2026-11-15 03:55:52,767] INFO : Plotted jets +rank0 [2026-11-15 03:55:53,788] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:53,808] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:55:55,211] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:55:55,662] INFO : Plotted number of elements +rank0 [2026-11-15 03:55:56,982] INFO : Plotted jets +rank0 [2026-11-15 03:55:57,975] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:58,381] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 03:55:58,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:59:51,752] INFO : Step 22000/500000 rank0 | Train Loss: 1.5651 | Train Loss Unweighted: 0.8086 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2504 | Regression_cos_phi: 0.0044 | Regression_energy: 0.2623 | Regression_eta: 0.0227 | Regression_pt: 0.2632 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1272 | Classification_binary: 0.2504 | Regression_cos_phi: 0.2581 | Regression_energy: 0.1570 | Regression_eta: 0.2943 | Regression_pt: 0.1574 | Regression_sin_phi: 0.3206 | LR: 4.50e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4057s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0544 | max_loss: 0.0029 | max_backward: 0.1171 | max_optimizer_step: 0.0735 | max_tensorboard_logging: 0.8662 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 6925.9 | uss_MB: 5717.1 | child_rss_MB: 23679.8 | allocated_MB: 196.4 | reserved_MB: 16168.0 | inactive_split_MB: 247.6 +rank0 [2026-11-15 03:59:51,754] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 03:59:51,821] INFO : Running validation on rank0 +rank0 [2026-11-15 03:59:51,821] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:59:51,821] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:59:52,445] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 03:59:52,564] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:59:52,677] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:59:52,746] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:59:52,813] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:59:52,879] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:59:52,946] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:59:53,012] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:59:53,122] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:59:53,225] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:59:53,327] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:59:53,394] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:59:53,461] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:59:53,527] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:59:53,593] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:59:53,660] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:59:53,763] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:59:53,782] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:59:53,795] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:59:53,798] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:59:53,808] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:59:53,819] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:59:53,821] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:59:53,823] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:59:53,826] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:59:53,828] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:59:53,836] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:59:53,840] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:59:53,842] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:59:53,845] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:59:53,848] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:59:53,850] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:00:01,548] INFO : VALIDATION | Step=22000/500000 | Train Loss=1.5651 | Valid Loss=2.8230 | Train Loss Unweighted=0.8086 | Valid Loss Unweighted=0.9650 | Stale=1 | ETA=2196.3m +rank0 [2026-11-15 04:00:01,548] INFO : split_configs=['1'] +rank0 [2026-11-15 04:00:01,559] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:00:01,566] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:00:01,650] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:02,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:02,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:03,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:03,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:04,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:04,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:04,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:05,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:05,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:05,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:06,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:06,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:06,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:07,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:07,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:07,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:07,793] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:00:07,797] INFO : split_configs=['1'] +rank0 [2026-11-15 04:00:07,808] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:00:07,811] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:00:07,869] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:08,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:09,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:09,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:10,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:10,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:11,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:11,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:11,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:12,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:12,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:13,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:13,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:14,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:14,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:15,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:15,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:15,635] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:00:15,638] INFO : split_configs=['1'] +rank0 [2026-11-15 04:00:15,649] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:00:15,652] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:00:15,711] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:16,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:17,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:17,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:17,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:18,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:18,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:19,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:19,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:19,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:20,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:20,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:21,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:21,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:22,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:22,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:22,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:22,930] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:00:22,938] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf +rank0 [2026-11-15 04:00:24,077] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf +rank0 [2026-11-15 04:00:24,507] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:26,225] INFO : Plotted jets +rank0 [2026-11-15 04:00:27,260] INFO : Plotted jet ratio +rank0 [2026-11-15 04:00:27,271] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:00:28,860] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:00:29,306] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:30,646] INFO : Plotted jets +rank0 [2026-11-15 04:00:31,671] INFO : Plotted jet ratio +rank0 [2026-11-15 04:00:31,682] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:00:33,114] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:00:33,569] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:34,903] INFO : Plotted jets +rank0 [2026-11-15 04:00:35,888] INFO : Plotted jet ratio +rank0 [2026-11-15 04:02:32,421] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 04:02:32,421] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:04:30,177] INFO : Step 23000/500000 rank0 | Train Loss: 2.4732 | Train Loss Unweighted: 0.9867 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2583 | Regression_cos_phi: 0.0069 | Regression_energy: 0.3323 | Regression_eta: 0.0427 | Regression_pt: 0.3340 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0934 | Classification_binary: 0.2583 | Regression_cos_phi: 0.4078 | Regression_energy: 0.1988 | Regression_eta: 0.5545 | Regression_pt: 0.1998 | Regression_sin_phi: 0.7605 | LR: 4.73e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4812s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0594 | max_loss: 0.0029 | max_backward: 0.1240 | max_optimizer_step: 0.0760 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7100.6 | uss_MB: 5891.8 | child_rss_MB: 23684.4 | allocated_MB: 197.0 | reserved_MB: 18782.0 | inactive_split_MB: 259.0 +rank0 [2026-11-15 04:04:30,179] INFO : Running validation on rank0 +rank0 [2026-11-15 04:04:30,179] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:04:30,179] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:04:30,812] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:04:30,939] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:04:31,054] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:04:31,125] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:04:31,191] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:04:31,257] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:04:31,323] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:04:31,391] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:04:31,497] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:04:31,603] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:04:31,708] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:04:31,778] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:04:31,846] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:04:31,914] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:04:31,982] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:04:32,051] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:04:32,157] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:04:32,177] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:04:32,189] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:04:32,193] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:04:32,202] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:04:32,215] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:04:32,218] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:04:32,228] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:04:32,241] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:04:32,244] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:04:32,246] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:04:32,249] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:04:32,251] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:04:32,253] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:04:32,256] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:04:32,258] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:04:39,974] INFO : VALIDATION | Step=23000/500000 | Train Loss=2.4732 | Valid Loss=2.7669 | Train Loss Unweighted=0.9867 | Valid Loss Unweighted=0.9500 | Stale=0 | ETA=2192.7m +rank0 [2026-11-15 04:04:39,974] INFO : split_configs=['1'] +rank0 [2026-11-15 04:04:39,985] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:04:39,990] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:04:40,070] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:40,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:41,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:41,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:41,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:42,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:42,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:42,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:43,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:43,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:43,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:44,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:44,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:44,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:04:45,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:04:45,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:04:45,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:04:46,057] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:04:46,060] INFO : split_configs=['1'] +rank0 [2026-11-15 04:04:46,071] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:04:46,074] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:04:46,125] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:47,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:47,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:47,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:48,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:48,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:49,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:49,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:50,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:50,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:51,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:51,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:51,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:52,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:04:52,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:04:53,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:04:53,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:04:53,791] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:04:53,794] INFO : split_configs=['1'] +rank0 [2026-11-15 04:04:53,805] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:04:53,809] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:04:53,861] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:54,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:55,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:55,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:56,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:56,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:56,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:57,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:57,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:58,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:58,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:58,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:59,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:59,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:05:00,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:05:00,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:05:00,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:05:01,017] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:05:01,025] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf +rank0 [2026-11-15 04:05:02,176] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf +rank0 [2026-11-15 04:05:02,611] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:03,885] INFO : Plotted jets +rank0 [2026-11-15 04:05:05,363] INFO : Plotted jet ratio +rank0 [2026-11-15 04:05:05,373] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:05:06,981] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:05:07,419] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:08,765] INFO : Plotted jets +rank0 [2026-11-15 04:05:09,791] INFO : Plotted jet ratio +rank0 [2026-11-15 04:05:09,801] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:05:11,213] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:05:11,653] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:12,970] INFO : Plotted jets +rank0 [2026-11-15 04:05:13,950] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:07,195] INFO : Step 24000/500000 rank0 | Train Loss: 3.6223 | Train Loss Unweighted: 1.0212 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2495 | Regression_cos_phi: 0.0183 | Regression_energy: 0.3425 | Regression_eta: 0.0739 | Regression_pt: 0.3233 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0999 | Classification_binary: 0.2495 | Regression_cos_phi: 1.0729 | Regression_energy: 0.2050 | Regression_eta: 0.9600 | Regression_pt: 0.1934 | Regression_sin_phi: 0.8418 | LR: 4.97e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4404s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0599 | max_loss: 0.0027 | max_backward: 0.1265 | max_optimizer_step: 0.0755 | max_tensorboard_logging: 0.8883 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7142.8 | uss_MB: 5933.9 | child_rss_MB: 23798.4 | allocated_MB: 197.5 | reserved_MB: 20284.0 | inactive_split_MB: 288.5 +rank0 [2026-11-15 04:09:07,197] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:09:07,261] INFO : Running validation on rank0 +rank0 [2026-11-15 04:09:07,261] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:09:07,261] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:09:07,883] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:09:08,004] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:09:08,124] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:09:08,194] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:09:08,262] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:09:08,329] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:09:08,395] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:09:08,462] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:09:08,569] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:09:08,671] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:09:08,772] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:09:08,839] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:09:08,905] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:09:08,969] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:09:09,035] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:09:09,101] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:09:09,201] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:09:09,208] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:09:09,211] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:09:09,213] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:09:09,216] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:09:09,219] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:09:09,221] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:09:09,223] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:09:09,226] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:09:09,228] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:09:09,231] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:09:09,233] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:09:09,236] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:09:09,238] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:09:09,241] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:09:09,243] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:09:16,955] INFO : VALIDATION | Step=24000/500000 | Train Loss=3.6223 | Valid Loss=2.7635 | Train Loss Unweighted=1.0212 | Valid Loss Unweighted=0.9409 | Stale=0 | ETA=2188.5m +rank0 [2026-11-15 04:09:16,955] INFO : split_configs=['1'] +rank0 [2026-11-15 04:09:16,966] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:09:16,969] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:09:17,061] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:09:17,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:09:18,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:09:18,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:09:18,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:09:19,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:09:19,647] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:09:19,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:09:20,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:09:20,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:09:20,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:09:21,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:09:21,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:09:22,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:09:22,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:09:22,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:09:22,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:09:23,053] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:09:23,056] INFO : split_configs=['1'] +rank0 [2026-11-15 04:09:23,067] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:09:23,074] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:09:23,143] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:09:24,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:09:24,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:09:25,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:09:25,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:09:25,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:09:26,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:09:26,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:09:27,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:09:27,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:09:28,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:09:28,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:09:29,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:09:29,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:09:29,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:09:30,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:09:30,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:09:30,927] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:09:30,930] INFO : split_configs=['1'] +rank0 [2026-11-15 04:09:30,940] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:09:30,943] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:09:31,004] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:09:32,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:09:32,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:09:32,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:09:33,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:09:33,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:09:34,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:09:34,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:09:34,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:09:35,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:09:35,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:09:36,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:09:36,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:09:36,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:09:37,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:09:37,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:09:38,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:09:38,252] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:09:38,259] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf +rank0 [2026-11-15 04:09:39,402] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf +rank0 [2026-11-15 04:09:39,827] INFO : Plotted number of elements +rank0 [2026-11-15 04:09:41,100] INFO : Plotted jets +rank0 [2026-11-15 04:09:42,550] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:42,563] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:09:44,125] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:09:44,570] INFO : Plotted number of elements +rank0 [2026-11-15 04:09:45,896] INFO : Plotted jets +rank0 [2026-11-15 04:09:46,914] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:46,923] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:09:48,323] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:09:48,765] INFO : Plotted number of elements +rank0 [2026-11-15 04:09:50,072] INFO : Plotted jets +rank0 [2026-11-15 04:09:51,053] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:51,460] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:09:51,460] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:13:44,448] INFO : Step 25000/500000 rank0 | Train Loss: 2.7232 | Train Loss Unweighted: 0.9901 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2429 | Regression_cos_phi: 0.0136 | Regression_energy: 0.3340 | Regression_eta: 0.0302 | Regression_pt: 0.3566 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2429 | Regression_cos_phi: 0.7996 | Regression_energy: 0.1999 | Regression_eta: 0.3929 | Regression_pt: 0.2133 | Regression_sin_phi: 0.7766 | LR: 5.22e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.5365s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0022 | max_forward: 0.0542 | max_loss: 0.0029 | max_backward: 0.1154 | max_optimizer_step: 0.0755 | max_tensorboard_logging: 0.9920 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 7240.9 | uss_MB: 6032.1 | child_rss_MB: 23713.7 | allocated_MB: 192.6 | reserved_MB: 18926.0 | inactive_split_MB: 171.4 +rank0 [2026-11-15 04:13:44,450] INFO : Running validation on rank0 +rank0 [2026-11-15 04:13:44,450] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:13:44,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:13:45,072] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:13:45,194] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:13:45,306] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:13:45,376] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:13:45,442] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:13:45,508] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:13:45,574] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:13:45,640] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:13:45,747] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:13:45,853] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:13:45,957] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:13:46,027] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:13:46,095] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:13:46,159] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:13:46,225] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:13:46,290] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:13:46,392] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:13:46,399] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:13:46,402] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:13:46,404] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:13:46,407] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:13:46,410] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:13:46,412] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:13:46,415] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:13:46,417] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:13:46,420] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:13:46,422] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:13:46,425] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:13:46,427] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:13:46,429] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:13:46,432] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:13:46,434] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:13:54,093] INFO : VALIDATION | Step=25000/500000 | Train Loss=2.7232 | Valid Loss=2.7764 | Train Loss Unweighted=0.9901 | Valid Loss Unweighted=0.9439 | Stale=1 | ETA=2184.3m +rank0 [2026-11-15 04:13:54,094] INFO : split_configs=['1'] +rank0 [2026-11-15 04:13:54,127] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:13:54,133] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:13:54,256] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:13:55,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:13:55,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:13:55,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:13:56,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:13:56,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:13:56,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:13:57,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:13:57,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:13:57,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:13:58,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:13:58,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:13:58,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:13:59,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:13:59,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:13:59,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:00,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:00,271] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:14:00,274] INFO : split_configs=['1'] +rank0 [2026-11-15 04:14:00,305] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:14:00,308] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:14:00,368] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:14:01,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:14:01,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:14:02,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:14:02,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:14:03,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:14:03,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:14:04,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:14:04,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:14:04,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:14:05,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:14:05,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:14:06,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:14:06,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:14:07,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:14:07,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:08,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:08,153] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:14:08,156] INFO : split_configs=['1'] +rank0 [2026-11-15 04:14:08,205] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:14:08,209] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:14:08,275] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:14:09,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:14:09,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:14:10,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:14:10,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:14:10,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:14:11,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:14:11,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:14:12,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:14:12,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:14:12,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:14:13,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:14:13,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:14:14,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:14:14,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:14:15,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:15,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:15,534] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:14:15,543] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf +rank0 [2026-11-15 04:14:16,684] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf +rank0 [2026-11-15 04:14:17,112] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:18,417] INFO : Plotted jets +rank0 [2026-11-15 04:14:19,896] INFO : Plotted jet ratio +rank0 [2026-11-15 04:14:19,908] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:14:21,496] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:14:21,951] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:23,291] INFO : Plotted jets +rank0 [2026-11-15 04:14:24,324] INFO : Plotted jet ratio +rank0 [2026-11-15 04:14:24,333] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:14:25,747] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:14:26,199] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:27,535] INFO : Plotted jets +rank0 [2026-11-15 04:14:28,560] INFO : Plotted jet ratio +rank0 [2026-11-15 04:16:25,059] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:16:25,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:22,664] INFO : Step 26000/500000 rank0 | Train Loss: 2.4374 | Train Loss Unweighted: 0.9093 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2384 | Regression_cos_phi: 0.0066 | Regression_energy: 0.3054 | Regression_eta: 0.0388 | Regression_pt: 0.3064 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.2384 | Regression_cos_phi: 0.3901 | Regression_energy: 0.1828 | Regression_eta: 0.5042 | Regression_pt: 0.1833 | Regression_sin_phi: 0.8374 | LR: 5.47e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4747s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0640 | max_loss: 0.0040 | max_backward: 0.1365 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.8381 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7251.5 | uss_MB: 6042.7 | child_rss_MB: 23874.5 | allocated_MB: 198.2 | reserved_MB: 16156.0 | inactive_split_MB: 149.8 +rank0 [2026-11-15 04:18:22,669] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:18:22,737] INFO : Running validation on rank0 +rank0 [2026-11-15 04:18:22,737] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:18:22,737] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:18:23,369] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:18:23,495] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:18:23,608] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:18:23,679] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:18:23,745] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:18:23,814] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:18:23,880] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:18:23,948] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:18:24,059] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:18:24,168] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:18:24,276] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:18:24,344] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:18:24,412] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:18:24,479] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:18:24,545] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:18:24,612] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:18:24,714] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:18:24,721] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:18:24,724] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:18:24,726] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:18:24,729] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:18:24,732] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:18:24,734] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:18:24,736] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:18:24,739] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:18:24,742] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:18:24,744] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:18:24,747] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:18:24,749] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:18:24,751] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:18:24,754] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:18:24,756] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:18:32,524] INFO : VALIDATION | Step=26000/500000 | Train Loss=2.4374 | Valid Loss=2.7336 | Train Loss Unweighted=0.9093 | Valid Loss Unweighted=0.9298 | Stale=0 | ETA=2180.4m +rank0 [2026-11-15 04:18:32,525] INFO : split_configs=['1'] +rank0 [2026-11-15 04:18:32,536] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:18:32,548] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:18:32,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:33,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:33,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:34,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:34,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:34,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:35,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:35,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:35,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:36,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:36,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:36,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:37,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:37,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:37,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:38,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:38,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:38,572] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:18:38,575] INFO : split_configs=['1'] +rank0 [2026-11-15 04:18:38,586] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:18:38,589] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:18:38,641] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:39,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:40,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:40,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:40,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:41,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:41,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:42,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:42,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:43,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:43,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:44,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:44,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:44,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:45,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:45,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:46,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:46,370] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:18:46,373] INFO : split_configs=['1'] +rank0 [2026-11-15 04:18:46,384] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:18:46,389] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:18:46,447] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:47,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:47,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:48,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:48,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:49,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:49,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:49,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:50,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:50,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:51,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:51,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:51,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:52,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:52,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:53,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:53,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:53,575] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 04:18:53,584] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf +rank0 [2026-11-15 04:18:54,717] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf +rank0 [2026-11-15 04:18:55,156] INFO : Plotted number of elements +rank0 [2026-11-15 04:18:56,882] INFO : Plotted jets +rank0 [2026-11-15 04:18:57,901] INFO : Plotted jet ratio +rank0 [2026-11-15 04:18:57,914] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:18:59,526] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:18:59,956] INFO : Plotted number of elements +rank0 [2026-11-15 04:19:01,299] INFO : Plotted jets +rank0 [2026-11-15 04:19:02,322] INFO : Plotted jet ratio +rank0 [2026-11-15 04:19:02,332] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:19:03,741] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:19:04,202] INFO : Plotted number of elements +rank0 [2026-11-15 04:19:05,512] INFO : Plotted jets +rank0 [2026-11-15 04:19:06,505] INFO : Plotted jet ratio +rank0 [2026-11-15 04:22:59,715] INFO : Step 27000/500000 rank0 | Train Loss: 2.2512 | Train Loss Unweighted: 0.9266 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0079 | Regression_energy: 0.3235 | Regression_eta: 0.0177 | Regression_pt: 0.3335 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2301 | Regression_cos_phi: 0.4662 | Regression_energy: 0.1936 | Regression_eta: 0.2299 | Regression_pt: 0.1994 | Regression_sin_phi: 0.8566 | LR: 5.74e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4271s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0029 | max_forward: 0.0637 | max_loss: 0.0028 | max_backward: 0.1336 | max_optimizer_step: 0.0755 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7396.7 | uss_MB: 6187.9 | child_rss_MB: 23664.2 | allocated_MB: 199.2 | reserved_MB: 18332.0 | inactive_split_MB: 322.8 +rank0 [2026-11-15 04:22:59,719] INFO : Running validation on rank0 +rank0 [2026-11-15 04:22:59,719] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:22:59,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:23:00,347] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:23:00,469] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:23:00,584] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:23:00,656] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:23:00,724] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:23:00,790] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:23:00,856] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:23:00,922] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:23:01,029] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:23:01,135] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:23:01,237] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:23:01,303] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:23:01,368] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:23:01,433] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:23:01,499] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:23:01,564] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:23:01,664] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:23:01,671] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:23:01,674] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:23:01,677] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:23:01,679] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:23:01,682] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:23:01,684] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:23:01,687] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:23:01,689] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:23:01,692] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:23:01,694] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:23:01,697] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:23:01,699] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:23:01,701] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:23:01,704] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:23:01,706] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:23:09,413] INFO : VALIDATION | Step=27000/500000 | Train Loss=2.2512 | Valid Loss=2.7552 | Train Loss Unweighted=0.9266 | Valid Loss Unweighted=0.9283 | Stale=1 | ETA=2176.1m +rank0 [2026-11-15 04:23:09,413] INFO : split_configs=['1'] +rank0 [2026-11-15 04:23:09,426] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:23:09,433] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:23:09,521] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:10,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:10,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:11,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:11,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:11,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:12,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:12,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:12,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:13,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:13,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:13,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:14,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:14,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:14,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:15,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:15,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:15,597] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:23:15,600] INFO : split_configs=['1'] +rank0 [2026-11-15 04:23:15,612] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:23:15,615] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:23:15,674] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:16,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:17,226] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:17,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:18,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:18,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:18,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:19,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:19,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:20,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:20,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:21,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:21,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:22,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:22,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:22,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:23,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:23,483] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:23:23,485] INFO : split_configs=['1'] +rank0 [2026-11-15 04:23:23,497] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:23:23,501] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:23:23,566] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:24,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:25,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:25,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:25,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:26,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:26,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:27,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:27,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:27,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:28,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:28,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:29,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:29,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:29,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:30,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:30,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:30,863] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:23:30,870] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf +rank0 [2026-11-15 04:23:32,015] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf +rank0 [2026-11-15 04:23:32,442] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:33,733] INFO : Plotted jets +rank0 [2026-11-15 04:23:35,222] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:35,234] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:23:36,834] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:23:37,273] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:38,614] INFO : Plotted jets +rank0 [2026-11-15 04:23:39,646] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:39,655] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:23:41,069] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:23:41,523] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:42,842] INFO : Plotted jets +rank0 [2026-11-15 04:23:43,838] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:44,259] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:23:44,260] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:37,525] INFO : Step 28000/500000 rank0 | Train Loss: 2.1634 | Train Loss Unweighted: 0.8074 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2405 | Regression_cos_phi: 0.0134 | Regression_energy: 0.2678 | Regression_eta: 0.0223 | Regression_pt: 0.2559 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.2405 | Regression_cos_phi: 0.7856 | Regression_energy: 0.1603 | Regression_eta: 0.2892 | Regression_pt: 0.1530 | Regression_sin_phi: 0.4535 | LR: 6.01e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4176s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0030 | max_forward: 0.0651 | max_loss: 0.0029 | max_backward: 0.1380 | max_optimizer_step: 0.0765 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7403.7 | uss_MB: 6194.9 | child_rss_MB: 23650.1 | allocated_MB: 200.6 | reserved_MB: 17570.0 | inactive_split_MB: 151.4 +rank0 [2026-11-15 04:27:37,528] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 04:27:37,596] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:37,596] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:37,596] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:38,219] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 04:27:38,338] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:38,449] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:38,518] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:38,584] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:38,648] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:38,714] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:38,780] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:38,889] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:38,994] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:39,101] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:39,170] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:39,237] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:39,304] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:39,370] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:39,436] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:39,537] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:39,544] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:39,546] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:39,549] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:39,551] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:39,554] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:39,556] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:39,559] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:39,561] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:39,564] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:39,566] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:39,569] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:39,571] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:39,573] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:39,576] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:39,578] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:47,285] INFO : VALIDATION | Step=28000/500000 | Train Loss=2.1634 | Valid Loss=2.7090 | Train Loss Unweighted=0.8074 | Valid Loss Unweighted=0.9227 | Stale=0 | ETA=2172.0m +rank0 [2026-11-15 04:27:47,287] INFO : split_configs=['1'] +rank0 [2026-11-15 04:27:47,300] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:27:47,308] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:27:47,396] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:48,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:48,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:48,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:49,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:49,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:49,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:50,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:50,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:50,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:51,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:51,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:52,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:52,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:52,647] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:52,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:53,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:53,375] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:27:53,378] INFO : split_configs=['1'] +rank0 [2026-11-15 04:27:53,389] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:27:53,392] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:27:53,450] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:54,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:54,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:55,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:55,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:56,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:56,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:57,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:57,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:58,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:58,465] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:58,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:59,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:59,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:00,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:00,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:01,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:01,210] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:28:01,215] INFO : split_configs=['1'] +rank0 [2026-11-15 04:28:01,225] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:28:01,229] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:28:01,290] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:28:02,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:28:02,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:28:03,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:28:03,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:28:03,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:28:04,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:28:04,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:28:05,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:28:05,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:28:05,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:28:06,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:28:06,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:28:07,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:07,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:08,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:08,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:08,513] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:28:08,520] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf +rank0 [2026-11-15 04:28:09,658] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf +rank0 [2026-11-15 04:28:10,086] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:11,832] INFO : Plotted jets +rank0 [2026-11-15 04:28:12,860] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:12,872] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:28:14,467] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:28:14,912] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:16,254] INFO : Plotted jets +rank0 [2026-11-15 04:28:17,278] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:17,287] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:28:18,748] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:28:19,207] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:20,525] INFO : Plotted jets +rank0 [2026-11-15 04:28:21,506] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:17,206] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:30:17,207] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:13,934] INFO : Step 29000/500000 rank0 | Train Loss: 2.3262 | Train Loss Unweighted: 0.8439 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2280 | Regression_cos_phi: 0.0126 | Regression_energy: 0.2810 | Regression_eta: 0.0211 | Regression_pt: 0.2906 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2280 | Regression_cos_phi: 0.7386 | Regression_energy: 0.1682 | Regression_eta: 0.2741 | Regression_pt: 0.1738 | Regression_sin_phi: 0.6456 | LR: 6.29e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.4801s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0612 | max_loss: 0.0029 | max_backward: 0.1305 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.9008 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 5804.6 | uss_MB: 4595.8 | child_rss_MB: 23672.4 | allocated_MB: 194.9 | reserved_MB: 18540.0 | inactive_split_MB: 287.1 +rank0 [2026-11-15 04:32:13,936] INFO : Running validation on rank0 +rank0 [2026-11-15 04:32:13,936] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:32:13,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:32:14,567] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 04:32:14,692] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:32:14,804] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:32:14,874] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:32:14,941] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:32:15,006] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:32:15,074] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:32:15,141] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:32:15,249] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:32:15,355] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:32:15,462] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:32:15,530] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:32:15,597] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:32:15,665] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:32:15,732] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:32:15,799] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:32:15,900] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:32:15,907] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:32:15,910] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:32:15,912] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:32:15,915] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:32:15,918] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:32:15,920] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:32:15,922] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:32:15,926] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:32:15,929] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:32:15,931] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:32:15,934] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:32:15,937] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:32:15,939] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:32:15,942] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:32:15,944] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:32:23,616] INFO : VALIDATION | Step=29000/500000 | Train Loss=2.3262 | Valid Loss=2.7383 | Train Loss Unweighted=0.8439 | Valid Loss Unweighted=0.9146 | Stale=1 | ETA=2167.5m +rank0 [2026-11-15 04:32:23,616] INFO : split_configs=['1'] +rank0 [2026-11-15 04:32:23,627] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:32:23,632] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:32:23,714] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:24,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:24,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:25,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:25,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:25,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:26,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:26,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:26,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:27,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:27,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:27,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:28,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:28,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:28,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:29,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:29,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:29,685] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:32:29,688] INFO : split_configs=['1'] +rank0 [2026-11-15 04:32:29,738] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:32:29,741] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:32:29,794] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:30,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:31,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:31,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:32,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:32,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:32,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:33,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:33,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:34,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:34,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:35,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:35,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:36,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:36,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:36,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:37,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:37,478] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:32:37,482] INFO : split_configs=['1'] +rank0 [2026-11-15 04:32:37,532] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:32:37,536] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:32:37,596] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:38,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:38,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:39,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:39,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:40,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:40,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:41,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:41,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:41,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:42,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:42,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:43,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:43,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:43,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:44,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:44,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:44,767] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 04:32:44,774] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf +rank0 [2026-11-15 04:32:45,924] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf +rank0 [2026-11-15 04:32:46,360] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:47,629] INFO : Plotted jets +rank0 [2026-11-15 04:32:49,098] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:49,109] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:32:50,738] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:32:51,172] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:52,496] INFO : Plotted jets +rank0 [2026-11-15 04:32:53,515] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:53,528] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:32:54,931] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:32:55,374] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:56,679] INFO : Plotted jets +rank0 [2026-11-15 04:32:57,664] INFO : Plotted jet ratio +rank0 [2026-11-15 04:36:51,240] INFO : Step 30000/500000 rank0 | Train Loss: 1.9500 | Train Loss Unweighted: 0.7760 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2434 | Regression_cos_phi: 0.0097 | Regression_energy: 0.2483 | Regression_eta: 0.0239 | Regression_pt: 0.2431 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2434 | Regression_cos_phi: 0.5689 | Regression_energy: 0.1486 | Regression_eta: 0.3108 | Regression_pt: 0.1454 | Regression_sin_phi: 0.4574 | LR: 6.58e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.5529s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0636 | max_loss: 0.0027 | max_backward: 0.1340 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.9691 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 7609.4 | uss_MB: 6400.6 | child_rss_MB: 23780.2 | allocated_MB: 198.4 | reserved_MB: 24478.0 | inactive_split_MB: 353.6 +rank0 [2026-11-15 04:36:51,242] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 04:36:51,308] INFO : Running validation on rank0 +rank0 [2026-11-15 04:36:51,308] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:36:51,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:36:51,934] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 04:36:52,056] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:36:52,175] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:36:52,246] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:36:52,312] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:36:52,377] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:36:52,443] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:36:52,510] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:36:52,618] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:36:52,722] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:36:52,825] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:36:52,892] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:36:52,958] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:36:53,023] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:36:53,091] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:36:53,157] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:36:53,259] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:36:53,282] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:36:53,295] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:36:53,298] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:36:53,308] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:36:53,320] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:36:53,324] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:36:53,365] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:36:53,368] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:36:53,370] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:36:53,372] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:36:53,375] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:36:53,377] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:36:53,380] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:36:53,383] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:36:53,385] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:01,052] INFO : VALIDATION | Step=30000/500000 | Train Loss=1.9500 | Valid Loss=2.7121 | Train Loss Unweighted=0.7760 | Valid Loss Unweighted=0.9101 | Stale=2 | ETA=2163.2m +rank0 [2026-11-15 04:37:01,052] INFO : split_configs=['1'] +rank0 [2026-11-15 04:37:01,093] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:37:01,100] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:37:01,199] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:02,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:02,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:02,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:03,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:03,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:03,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:04,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:04,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:04,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:05,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:05,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:05,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:06,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:06,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:06,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:07,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:07,233] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:37:07,236] INFO : split_configs=['1'] +rank0 [2026-11-15 04:37:07,247] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:37:07,252] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:37:07,313] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:08,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:08,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:09,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:09,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:10,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:10,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:10,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:11,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:11,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:12,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:12,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:13,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:13,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:14,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:14,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:15,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:15,083] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:37:15,087] INFO : split_configs=['1'] +rank0 [2026-11-15 04:37:15,098] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:37:15,101] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:37:15,162] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:16,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:16,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:17,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:17,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:17,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:18,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:18,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:19,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:19,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:19,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:20,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:20,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:21,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:21,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:21,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:22,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:22,410] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:37:22,418] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf +rank0 [2026-11-15 04:37:23,546] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf +rank0 [2026-11-15 04:37:23,965] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:25,232] INFO : Plotted jets +rank0 [2026-11-15 04:37:26,708] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:26,721] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:37:28,340] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:37:28,775] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:30,109] INFO : Plotted jets +rank0 [2026-11-15 04:37:31,132] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:31,142] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:37:32,535] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:37:32,975] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:34,288] INFO : Plotted jets +rank0 [2026-11-15 04:37:35,273] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:35,693] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:37:35,693] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:29,265] INFO : Step 31000/500000 rank0 | Train Loss: 3.0354 | Train Loss Unweighted: 1.0306 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2339 | Regression_cos_phi: 0.0087 | Regression_energy: 0.3662 | Regression_eta: 0.0801 | Regression_pt: 0.3297 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1029 | Classification_binary: 0.2339 | Regression_cos_phi: 0.5129 | Regression_energy: 0.2191 | Regression_eta: 1.0412 | Regression_pt: 0.1972 | Regression_sin_phi: 0.7282 | LR: 6.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.5941s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0559 | max_loss: 0.0030 | max_backward: 0.1172 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 1.0237 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7607.0 | uss_MB: 6461.5 | child_rss_MB: 23805.7 | allocated_MB: 194.6 | reserved_MB: 22638.0 | inactive_split_MB: 297.4 +rank0 [2026-11-15 04:41:29,269] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:29,269] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:29,269] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:29,901] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 04:41:30,024] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:30,143] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:30,214] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:30,280] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:30,345] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:30,412] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:30,479] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:30,587] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:30,692] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:30,793] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:30,862] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:30,928] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:30,993] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:31,061] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:31,128] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:31,230] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:31,253] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:31,266] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:31,269] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:31,279] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:31,292] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:31,295] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:31,305] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:31,318] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:31,321] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:31,331] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:31,340] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:31,342] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:31,344] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:31,347] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:31,350] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:39,025] INFO : VALIDATION | Step=31000/500000 | Train Loss=3.0354 | Valid Loss=2.7676 | Train Loss Unweighted=1.0306 | Valid Loss Unweighted=0.9194 | Stale=3 | ETA=2159.1m +rank0 [2026-11-15 04:41:39,026] INFO : split_configs=['1'] +rank0 [2026-11-15 04:41:39,073] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:41:39,081] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:41:39,173] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:40,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:40,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:40,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:41,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:41,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:41,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:42,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:42,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:42,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:43,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:43,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:43,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:44,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:44,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:44,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:45,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:45,221] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:41:45,224] INFO : split_configs=['1'] +rank0 [2026-11-15 04:41:45,265] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:41:45,269] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:41:45,330] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:46,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:46,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:47,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:47,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:48,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:48,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:48,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:49,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:49,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:50,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:50,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:51,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:51,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:52,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:52,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:53,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:53,079] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:41:53,083] INFO : split_configs=['1'] +rank0 [2026-11-15 04:41:53,121] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:41:53,125] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:41:53,187] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:54,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:54,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:55,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:55,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:55,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:56,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:56,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:57,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:57,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:57,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:58,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:58,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:59,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:59,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:59,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:42:00,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:42:00,414] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:42:00,421] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf +rank0 [2026-11-15 04:42:01,577] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf +rank0 [2026-11-15 04:42:02,004] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:03,301] INFO : Plotted jets +rank0 [2026-11-15 04:42:04,800] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:04,812] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:42:06,396] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:42:06,831] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:08,181] INFO : Plotted jets +rank0 [2026-11-15 04:42:09,202] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:09,211] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:42:10,618] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:42:11,086] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:12,396] INFO : Plotted jets +rank0 [2026-11-15 04:42:13,379] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:09,650] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:44:09,650] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:07,052] INFO : Step 32000/500000 rank0 | Train Loss: 2.0221 | Train Loss Unweighted: 0.7242 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2300 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2261 | Regression_eta: 0.0333 | Regression_pt: 0.2180 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0889 | Classification_binary: 0.2300 | Regression_cos_phi: 0.4763 | Regression_energy: 0.1353 | Regression_eta: 0.4320 | Regression_pt: 0.1304 | Regression_sin_phi: 0.5293 | LR: 7.19e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.6176s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0733 | max_loss: 0.0028 | max_backward: 0.1612 | max_optimizer_step: 0.0749 | max_tensorboard_logging: 1.0033 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7422.0 | uss_MB: 6276.5 | child_rss_MB: 23817.2 | allocated_MB: 198.6 | reserved_MB: 19218.0 | inactive_split_MB: 569.4 +rank0 [2026-11-15 04:46:07,054] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 04:46:07,127] INFO : Running validation on rank0 +rank0 [2026-11-15 04:46:07,128] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:46:07,128] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:46:07,749] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 04:46:07,870] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:46:07,985] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:46:08,057] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:46:08,125] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:46:08,190] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:46:08,257] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:46:08,323] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:46:08,430] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:46:08,533] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:46:08,635] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:46:08,702] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:46:08,768] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:46:08,833] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:46:08,898] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:46:08,963] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:46:09,066] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:46:09,091] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:46:09,104] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:46:09,107] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:46:09,117] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:46:09,130] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:46:09,133] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:46:09,143] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:46:09,156] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:46:09,172] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:46:09,174] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:46:09,175] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:46:09,176] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:46:09,177] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:46:09,178] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:46:09,179] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:46:16,870] INFO : VALIDATION | Step=32000/500000 | Train Loss=2.0221 | Valid Loss=2.7485 | Train Loss Unweighted=0.7242 | Valid Loss Unweighted=0.9045 | Stale=4 | ETA=2154.9m +rank0 [2026-11-15 04:46:16,870] INFO : split_configs=['1'] +rank0 [2026-11-15 04:46:16,912] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:46:16,921] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:46:17,010] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:46:17,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:18,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:18,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:18,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:19,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:19,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:19,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:20,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:20,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:20,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:21,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:21,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:21,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:22,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:22,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:22,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:23,023] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:46:23,026] INFO : split_configs=['1'] +rank0 [2026-11-15 04:46:23,062] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:46:23,065] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:46:23,126] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:46:24,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:24,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:25,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:25,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:25,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:26,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:26,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:27,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:27,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:28,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:28,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:29,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:29,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:29,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:30,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:30,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:30,901] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:46:30,904] INFO : split_configs=['1'] +rank0 [2026-11-15 04:46:30,958] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:46:30,961] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:46:31,027] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:46:32,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:32,465] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:32,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:33,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:33,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:34,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:34,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:34,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:35,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:35,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:36,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:36,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:36,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:37,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:37,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:38,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:38,273] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:46:38,280] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf +rank0 [2026-11-15 04:46:39,415] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf +rank0 [2026-11-15 04:46:39,831] INFO : Plotted number of elements +rank0 [2026-11-15 04:46:41,116] INFO : Plotted jets +rank0 [2026-11-15 04:46:42,584] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:42,596] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:46:44,181] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:46:44,616] INFO : Plotted number of elements +rank0 [2026-11-15 04:46:45,939] INFO : Plotted jets +rank0 [2026-11-15 04:46:46,962] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:46,971] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:46:48,373] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:46:48,816] INFO : Plotted number of elements +rank0 [2026-11-15 04:46:50,122] INFO : Plotted jets +rank0 [2026-11-15 04:46:51,105] INFO : Plotted jet ratio +rank0 [2026-11-15 04:50:44,246] INFO : Step 33000/500000 rank0 | Train Loss: 2.7063 | Train Loss Unweighted: 0.9333 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2388 | Regression_cos_phi: 0.0118 | Regression_energy: 0.3127 | Regression_eta: 0.0551 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1030 | Classification_binary: 0.2388 | Regression_cos_phi: 0.6910 | Regression_energy: 0.1871 | Regression_eta: 0.7161 | Regression_pt: 0.1825 | Regression_sin_phi: 0.5878 | LR: 7.51e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.5544s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0024 | max_forward: 0.0588 | max_loss: 0.0027 | max_backward: 0.1248 | max_optimizer_step: 0.0881 | max_tensorboard_logging: 0.9698 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 7835.4 | uss_MB: 6689.9 | child_rss_MB: 23727.0 | allocated_MB: 195.7 | reserved_MB: 19098.0 | inactive_split_MB: 108.3 +rank0 [2026-11-15 04:50:44,251] INFO : Running validation on rank0 +rank0 [2026-11-15 04:50:44,251] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:50:44,251] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:50:44,882] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 04:50:45,002] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:50:45,118] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:50:45,188] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:50:45,255] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:50:45,321] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:50:45,388] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:50:45,455] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:50:45,566] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:50:45,670] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:50:45,772] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:50:45,841] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:50:45,909] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:50:45,976] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:50:46,045] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:50:46,114] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:50:46,218] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:50:46,225] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:50:46,240] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:50:46,241] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:50:46,243] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:50:46,244] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:50:46,245] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:50:46,252] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:50:46,254] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:50:46,255] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:50:46,256] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:50:46,257] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:50:46,258] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:50:46,259] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:50:46,260] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:50:46,261] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:50:54,032] INFO : VALIDATION | Step=33000/500000 | Train Loss=2.7063 | Valid Loss=2.6776 | Train Loss Unweighted=0.9333 | Valid Loss Unweighted=0.8917 | Stale=0 | ETA=2150.5m +rank0 [2026-11-15 04:50:54,033] INFO : split_configs=['1'] +rank0 [2026-11-15 04:50:54,074] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:50:54,082] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:50:54,177] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:50:55,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:50:55,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:50:55,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:50:56,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:50:56,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:50:56,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:50:57,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:50:57,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:50:57,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:50:58,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:50:58,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:50:58,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:50:59,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:50:59,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:50:59,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:00,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:00,241] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:51:00,244] INFO : split_configs=['1'] +rank0 [2026-11-15 04:51:00,285] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:51:00,289] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:51:00,351] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:01,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:01,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:02,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:02,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:03,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:03,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:04,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:04,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:04,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:05,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:05,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:06,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:06,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:07,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:07,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:08,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:08,142] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:51:08,145] INFO : split_configs=['1'] +rank0 [2026-11-15 04:51:08,185] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:51:08,189] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:51:08,253] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:09,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:09,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:10,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:10,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:10,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:11,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:11,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:12,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:12,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:13,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:13,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:13,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:14,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:14,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:15,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:15,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:15,602] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:51:15,609] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf +rank0 [2026-11-15 04:51:16,751] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf +rank0 [2026-11-15 04:51:17,179] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:18,485] INFO : Plotted jets +rank0 [2026-11-15 04:51:19,974] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:19,986] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:51:21,569] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:51:22,004] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:23,356] INFO : Plotted jets +rank0 [2026-11-15 04:51:24,382] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:24,392] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:51:25,801] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:51:26,266] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:27,581] INFO : Plotted jets +rank0 [2026-11-15 04:51:28,581] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:29,008] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:51:29,008] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:55:22,934] INFO : Step 34000/500000 rank0 | Train Loss: 1.8600 | Train Loss Unweighted: 0.9171 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2606 | Regression_cos_phi: 0.0069 | Regression_energy: 0.3089 | Regression_eta: 0.0237 | Regression_pt: 0.3102 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1140 | Classification_binary: 0.2606 | Regression_cos_phi: 0.4077 | Regression_energy: 0.1848 | Regression_eta: 0.3083 | Regression_pt: 0.1855 | Regression_sin_phi: 0.3992 | LR: 7.83e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.4464s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0658 | max_loss: 0.0030 | max_backward: 0.1392 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7918.8 | uss_MB: 6773.3 | child_rss_MB: 23850.9 | allocated_MB: 199.5 | reserved_MB: 18038.0 | inactive_split_MB: 312.5 +rank0 [2026-11-15 04:55:22,937] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 04:55:23,005] INFO : Running validation on rank0 +rank0 [2026-11-15 04:55:23,005] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:55:23,005] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:55:23,626] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 04:55:23,746] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:55:23,858] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:55:23,927] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:55:23,994] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:55:24,061] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:55:24,129] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:55:24,197] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:55:24,307] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:55:24,413] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:55:24,519] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:55:24,590] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:55:24,658] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:55:24,726] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:55:24,793] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:55:24,859] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:55:24,962] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:55:24,969] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:55:24,972] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:55:24,974] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:55:24,977] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:55:24,979] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:55:24,982] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:55:24,984] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:55:24,987] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:55:24,989] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:55:24,991] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:55:24,994] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:55:24,997] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:55:25,040] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:55:25,044] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:55:25,046] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:55:32,801] INFO : VALIDATION | Step=34000/500000 | Train Loss=1.8600 | Valid Loss=2.6758 | Train Loss Unweighted=0.9171 | Valid Loss Unweighted=0.9024 | Stale=0 | ETA=2146.4m +rank0 [2026-11-15 04:55:32,802] INFO : split_configs=['1'] +rank0 [2026-11-15 04:55:32,813] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:55:32,833] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:55:32,917] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:33,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:34,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:34,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:34,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:35,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:35,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:35,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:36,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:36,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:36,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:37,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:37,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:37,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:38,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:38,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:38,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:38,948] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:55:38,951] INFO : split_configs=['1'] +rank0 [2026-11-15 04:55:38,963] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:55:38,966] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:55:39,025] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:40,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:40,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:40,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:41,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:41,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:42,236] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:42,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:43,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:43,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:44,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:44,456] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:44,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:45,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:45,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:46,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:46,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:46,731] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:55:46,734] INFO : split_configs=['1'] +rank0 [2026-11-15 04:55:46,745] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:55:46,749] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:55:46,809] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:47,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:48,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:48,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:49,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:49,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:49,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:50,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:50,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:51,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:51,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:51,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:52,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:52,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:53,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:53,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:53,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:54,064] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:55:54,072] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf +rank0 [2026-11-15 04:55:55,223] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf +rank0 [2026-11-15 04:55:55,674] INFO : Plotted number of elements +rank0 [2026-11-15 04:55:56,964] INFO : Plotted jets +rank0 [2026-11-15 04:55:58,479] INFO : Plotted jet ratio +rank0 [2026-11-15 04:55:58,490] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:56:00,092] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:56:00,545] INFO : Plotted number of elements +rank0 [2026-11-15 04:56:01,890] INFO : Plotted jets +rank0 [2026-11-15 04:56:02,927] INFO : Plotted jet ratio +rank0 [2026-11-15 04:56:02,938] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:56:04,365] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:56:04,817] INFO : Plotted number of elements +rank0 [2026-11-15 04:56:06,145] INFO : Plotted jets +rank0 [2026-11-15 04:56:07,145] INFO : Plotted jet ratio +rank0 [2026-11-15 04:58:03,514] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:58:03,514] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:00:00,551] INFO : Step 35000/500000 rank0 | Train Loss: 1.6872 | Train Loss Unweighted: 0.8917 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2617 | Regression_cos_phi: 0.0053 | Regression_energy: 0.2936 | Regression_eta: 0.0196 | Regression_pt: 0.3045 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2617 | Regression_cos_phi: 0.3132 | Regression_energy: 0.1757 | Regression_eta: 0.2545 | Regression_pt: 0.1821 | Regression_sin_phi: 0.4228 | LR: 8.16e-05 | DataLoad Time: 0.0023s | Model Forward Time: 1.5350s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0029 | max_forward: 0.0766 | max_loss: 0.0033 | max_backward: 0.1700 | max_optimizer_step: 0.0795 | max_tensorboard_logging: 0.8781 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7895.6 | uss_MB: 6750.1 | child_rss_MB: 23888.8 | allocated_MB: 198.8 | reserved_MB: 18350.0 | inactive_split_MB: 293.2 +rank0 [2026-11-15 05:00:00,553] INFO : Running validation on rank0 +rank0 [2026-11-15 05:00:00,553] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:00:00,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:00:01,175] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 05:00:01,297] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:00:01,408] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:00:01,478] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:00:01,544] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:00:01,610] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:00:01,677] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:00:01,745] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:00:01,855] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:00:01,962] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:00:02,068] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:00:02,137] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:00:02,204] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:00:02,270] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:00:02,336] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:00:02,402] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:00:02,504] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:00:02,512] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:00:02,515] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:00:02,517] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:00:02,520] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:00:02,522] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:00:02,525] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:00:02,527] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:00:02,530] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:00:02,532] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:00:02,534] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:00:02,537] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:00:02,539] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:00:02,542] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:00:02,546] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:00:02,548] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:00:10,277] INFO : VALIDATION | Step=35000/500000 | Train Loss=1.6872 | Valid Loss=2.6696 | Train Loss Unweighted=0.8917 | Valid Loss Unweighted=0.8862 | Stale=0 | ETA=2142.1m +rank0 [2026-11-15 05:00:10,277] INFO : split_configs=['1'] +rank0 [2026-11-15 05:00:10,288] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:00:10,296] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:00:10,387] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:00:11,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:00:11,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:00:11,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:00:12,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:00:12,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:00:13,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:00:13,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:00:13,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:00:14,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:00:14,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:00:14,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:00:15,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:00:15,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:00:15,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:00:15,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:00:16,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:00:16,445] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:00:16,448] INFO : split_configs=['1'] +rank0 [2026-11-15 05:00:16,460] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:00:16,464] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:00:16,528] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:00:17,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:00:18,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:00:18,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:00:18,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:00:19,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:00:19,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:00:20,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:00:20,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:00:21,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:00:21,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:00:22,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:00:22,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:00:22,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:00:23,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:00:23,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:00:24,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:00:24,366] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:00:24,369] INFO : split_configs=['1'] +rank0 [2026-11-15 05:00:24,381] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:00:24,384] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:00:24,449] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:00:25,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:00:25,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:00:26,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:00:26,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:00:27,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:00:27,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:00:27,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:00:28,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:00:28,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:00:29,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:00:29,615] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:00:30,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:00:30,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:00:30,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:00:31,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:00:31,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:00:31,768] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:00:31,788] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf +rank0 [2026-11-15 05:00:32,958] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf +rank0 [2026-11-15 05:00:33,388] INFO : Plotted number of elements +rank0 [2026-11-15 05:00:34,684] INFO : Plotted jets +rank0 [2026-11-15 05:00:36,195] INFO : Plotted jet ratio +rank0 [2026-11-15 05:00:36,207] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:00:37,787] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:00:38,223] INFO : Plotted number of elements +rank0 [2026-11-15 05:00:39,569] INFO : Plotted jets +rank0 [2026-11-15 05:00:40,606] INFO : Plotted jet ratio +rank0 [2026-11-15 05:00:40,614] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:00:42,023] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:00:42,476] INFO : Plotted number of elements +rank0 [2026-11-15 05:00:43,787] INFO : Plotted jets +rank0 [2026-11-15 05:00:44,784] INFO : Plotted jet ratio +rank0 [2026-11-15 05:04:39,262] INFO : Step 36000/500000 rank0 | Train Loss: 1.6653 | Train Loss Unweighted: 0.8876 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2262 | Regression_cos_phi: 0.0049 | Regression_energy: 0.3084 | Regression_eta: 0.0170 | Regression_pt: 0.3235 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1020 | Classification_binary: 0.2262 | Regression_cos_phi: 0.2849 | Regression_energy: 0.1845 | Regression_eta: 0.2213 | Regression_pt: 0.1935 | Regression_sin_phi: 0.4529 | LR: 8.50e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4649s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0592 | max_loss: 0.0033 | max_backward: 0.1277 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7969.3 | uss_MB: 6823.8 | child_rss_MB: 23809.6 | allocated_MB: 195.4 | reserved_MB: 20320.0 | inactive_split_MB: 186.6 +rank0 [2026-11-15 05:04:39,264] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-36000.pth +rank0 [2026-11-15 05:04:39,332] INFO : Running validation on rank0 +rank0 [2026-11-15 05:04:39,333] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:04:39,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:04:39,962] INFO : saved batch0_epoch36000.parquet +rank0 [2026-11-15 05:04:40,086] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:04:40,200] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:04:40,269] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:04:40,337] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:04:40,403] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:04:40,470] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:04:40,539] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:04:40,650] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:04:40,758] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:04:40,864] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:04:40,935] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:04:41,004] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:04:41,073] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:04:41,141] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:04:41,208] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:04:41,309] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:04:41,316] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:04:41,319] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:04:41,321] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:04:41,324] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:04:41,328] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:04:41,330] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:04:41,332] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:04:41,336] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:04:41,338] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:04:41,341] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:04:41,343] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:04:41,346] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:04:41,348] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:04:41,351] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:04:41,353] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:04:49,141] INFO : VALIDATION | Step=36000/500000 | Train Loss=1.6653 | Valid Loss=2.6418 | Train Loss Unweighted=0.8876 | Valid Loss Unweighted=0.8849 | Stale=0 | ETA=2138.0m +rank0 [2026-11-15 05:04:49,142] INFO : split_configs=['1'] +rank0 [2026-11-15 05:04:49,153] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:04:49,162] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:04:49,255] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:04:50,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:04:50,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:04:50,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:04:51,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:04:51,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:04:52,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:04:52,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:04:52,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:04:53,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:04:53,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:04:53,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:04:54,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:04:54,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:04:54,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:04:55,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:04:55,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:04:55,709] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:04:55,712] INFO : split_configs=['1'] +rank0 [2026-11-15 05:04:55,723] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:04:55,729] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:04:55,794] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:04:56,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:04:57,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:04:57,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:04:58,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:04:58,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:04:59,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:04:59,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:05:00,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:05:00,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:05:01,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:05:01,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:05:02,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:05:02,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:05:03,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:05:03,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:05:04,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:05:04,103] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:05:04,108] INFO : split_configs=['1'] +rank0 [2026-11-15 05:05:04,120] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:05:04,123] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:05:04,193] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:05:05,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:05:05,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:05:06,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:05:06,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:05:06,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:05:07,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:05:07,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:05:08,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:05:08,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:05:08,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:05:09,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:05:09,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:05:10,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:05:10,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:05:11,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:05:11,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:05:11,521] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:05:11,530] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf +rank0 [2026-11-15 05:05:12,693] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf +rank0 [2026-11-15 05:05:13,121] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:14,412] INFO : Plotted jets +rank0 [2026-11-15 05:05:15,889] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:15,911] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:05:17,534] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:05:17,967] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:19,307] INFO : Plotted jets +rank0 [2026-11-15 05:05:20,325] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:20,334] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:05:21,758] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:05:22,216] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:23,518] INFO : Plotted jets +rank0 [2026-11-15 05:05:24,497] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:24,911] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 05:05:24,911] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:09:17,756] INFO : Step 37000/500000 rank0 | Train Loss: 2.6032 | Train Loss Unweighted: 0.9198 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2519 | Regression_cos_phi: 0.0118 | Regression_energy: 0.3004 | Regression_eta: 0.0590 | Regression_pt: 0.2894 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0978 | Classification_binary: 0.2519 | Regression_cos_phi: 0.6929 | Regression_energy: 0.1798 | Regression_eta: 0.7659 | Regression_pt: 0.1731 | Regression_sin_phi: 0.4418 | LR: 8.85e-05 | DataLoad Time: 0.0017s | Model Forward Time: 1.4264s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0561 | max_loss: 0.0029 | max_backward: 0.1173 | max_optimizer_step: 0.0801 | max_tensorboard_logging: 0.8205 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8132.9 | uss_MB: 6987.5 | child_rss_MB: 23797.0 | allocated_MB: 196.0 | reserved_MB: 21064.0 | inactive_split_MB: 400.0 +rank0 [2026-11-15 05:09:17,799] INFO : Running validation on rank0 +rank0 [2026-11-15 05:09:17,799] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:09:17,799] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:09:18,430] INFO : saved batch0_epoch37000.parquet +rank0 [2026-11-15 05:09:18,556] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:09:18,671] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:09:18,740] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:09:18,805] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:09:18,871] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:09:18,938] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:09:19,004] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:09:19,117] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:09:19,222] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:09:19,328] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:09:19,398] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:09:19,466] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:09:19,534] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:09:19,603] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:09:19,669] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:09:19,773] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:09:19,780] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:09:19,783] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:09:19,785] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:09:19,788] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:09:19,792] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:09:19,794] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:09:19,797] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:09:19,799] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:09:19,802] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:09:19,804] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:09:19,807] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:09:19,809] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:09:19,811] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:09:19,814] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:09:19,816] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:09:27,488] INFO : VALIDATION | Step=37000/500000 | Train Loss=2.6032 | Valid Loss=2.6474 | Train Loss Unweighted=0.9198 | Valid Loss Unweighted=0.8803 | Stale=1 | ETA=2133.8m +rank0 [2026-11-15 05:09:27,489] INFO : split_configs=['1'] +rank0 [2026-11-15 05:09:27,500] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:09:27,508] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:09:27,599] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:09:28,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:09:28,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:09:29,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:09:29,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:09:29,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:09:30,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:09:30,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:09:30,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:09:31,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:09:31,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:09:31,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:09:32,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:09:32,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:09:32,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:09:33,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:09:33,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:09:33,627] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:09:33,630] INFO : split_configs=['1'] +rank0 [2026-11-15 05:09:33,643] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:09:33,646] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:09:33,707] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:09:34,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:09:35,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:09:35,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:09:36,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:09:36,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:09:36,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:09:37,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:09:37,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:09:38,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:09:38,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:09:39,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:09:39,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:09:40,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:09:40,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:09:40,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:09:41,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:09:41,427] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:09:41,430] INFO : split_configs=['1'] +rank0 [2026-11-15 05:09:41,441] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:09:41,445] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:09:41,510] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:09:42,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:09:42,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:09:43,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:09:43,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:09:44,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:09:44,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:09:44,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:09:45,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:09:45,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:09:46,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:09:46,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:09:46,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:09:47,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:09:47,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:09:48,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:09:48,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:09:48,695] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 05:09:48,702] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf +rank0 [2026-11-15 05:09:49,854] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf +rank0 [2026-11-15 05:09:50,289] INFO : Plotted number of elements +rank0 [2026-11-15 05:09:51,574] INFO : Plotted jets +rank0 [2026-11-15 05:09:53,086] INFO : Plotted jet ratio +rank0 [2026-11-15 05:09:53,098] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:09:54,738] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:09:55,188] INFO : Plotted number of elements +rank0 [2026-11-15 05:09:56,540] INFO : Plotted jets +rank0 [2026-11-15 05:09:57,570] INFO : Plotted jet ratio +rank0 [2026-11-15 05:09:57,579] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:09:59,091] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:09:59,538] INFO : Plotted number of elements +rank0 [2026-11-15 05:10:00,859] INFO : Plotted jets +rank0 [2026-11-15 05:10:01,846] INFO : Plotted jet ratio +rank0 [2026-11-15 05:11:58,720] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 05:11:58,720] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:13:55,956] INFO : Step 38000/500000 rank0 | Train Loss: 2.1653 | Train Loss Unweighted: 0.8310 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2411 | Regression_cos_phi: 0.0056 | Regression_energy: 0.2672 | Regression_eta: 0.0563 | Regression_pt: 0.2532 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1003 | Classification_binary: 0.2411 | Regression_cos_phi: 0.3311 | Regression_energy: 0.1599 | Regression_eta: 0.7318 | Regression_pt: 0.1515 | Regression_sin_phi: 0.4497 | LR: 9.21e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.4117s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0021 | max_forward: 0.0516 | max_loss: 0.0029 | max_backward: 0.1099 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.8435 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8083.8 | uss_MB: 6938.5 | child_rss_MB: 23850.3 | allocated_MB: 190.0 | reserved_MB: 20456.0 | inactive_split_MB: 218.0 +rank0 [2026-11-15 05:13:55,959] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-38000.pth +rank0 [2026-11-15 05:13:56,027] INFO : Running validation on rank0 +rank0 [2026-11-15 05:13:56,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:13:56,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:13:56,645] INFO : saved batch0_epoch38000.parquet +rank0 [2026-11-15 05:13:56,768] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:13:56,882] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:13:56,953] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:13:57,021] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:13:57,091] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:13:57,160] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:13:57,230] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:13:57,337] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:13:57,442] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:13:57,544] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:13:57,612] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:13:57,678] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:13:57,743] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:13:57,809] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:13:57,873] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:13:57,974] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:13:57,981] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:13:57,984] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:13:57,986] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:13:57,989] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:13:57,992] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:13:57,994] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:13:57,996] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:13:57,999] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:13:58,001] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:13:58,004] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:13:58,006] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:13:58,009] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:13:58,011] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:13:58,014] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:13:58,016] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:14:05,728] INFO : VALIDATION | Step=38000/500000 | Train Loss=2.1653 | Valid Loss=2.6244 | Train Loss Unweighted=0.8310 | Valid Loss Unweighted=0.8814 | Stale=0 | ETA=2129.5m +rank0 [2026-11-15 05:14:05,729] INFO : split_configs=['1'] +rank0 [2026-11-15 05:14:05,746] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:14:05,753] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:14:05,847] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:14:06,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:14:07,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:14:07,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:14:07,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:14:08,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:14:08,456] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:14:08,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:14:09,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:14:09,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:14:09,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:14:10,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:14:10,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:14:10,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:14:11,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:14:11,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:14:11,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:14:11,901] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:14:11,904] INFO : split_configs=['1'] +rank0 [2026-11-15 05:14:11,916] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:14:11,921] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:14:11,987] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:14:13,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:14:13,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:14:13,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:14:14,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:14:14,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:14:15,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:14:15,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:14:16,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:14:16,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:14:17,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:14:17,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:14:17,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:14:18,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:14:18,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:14:19,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:14:19,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:14:19,794] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:14:19,798] INFO : split_configs=['1'] +rank0 [2026-11-15 05:14:19,809] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:14:19,813] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:14:19,879] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:14:20,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:14:21,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:14:21,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:14:22,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:14:22,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:14:22,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:14:23,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:14:23,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:14:24,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:14:24,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:14:25,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:14:25,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:14:25,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:14:26,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:14:26,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:14:27,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:14:27,160] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:14:27,168] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf +rank0 [2026-11-15 05:14:28,328] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf +rank0 [2026-11-15 05:14:28,752] INFO : Plotted number of elements +rank0 [2026-11-15 05:14:30,042] INFO : Plotted jets +rank0 [2026-11-15 05:14:31,544] INFO : Plotted jet ratio +rank0 [2026-11-15 05:14:31,556] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:14:33,151] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:14:33,594] INFO : Plotted number of elements +rank0 [2026-11-15 05:14:34,928] INFO : Plotted jets +rank0 [2026-11-15 05:14:35,961] INFO : Plotted jet ratio +rank0 [2026-11-15 05:14:35,970] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:14:37,406] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:14:37,859] INFO : Plotted number of elements +rank0 [2026-11-15 05:14:39,178] INFO : Plotted jets +rank0 [2026-11-15 05:14:40,166] INFO : Plotted jet ratio +rank0 [2026-11-15 05:18:33,321] INFO : Step 39000/500000 rank0 | Train Loss: 2.1727 | Train Loss Unweighted: 0.9065 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2366 | Regression_cos_phi: 0.0088 | Regression_energy: 0.3197 | Regression_eta: 0.0391 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1076 | Classification_binary: 0.2366 | Regression_cos_phi: 0.5173 | Regression_energy: 0.1913 | Regression_eta: 0.5076 | Regression_pt: 0.1764 | Regression_sin_phi: 0.4359 | LR: 9.57e-05 | DataLoad Time: 0.0148s | Model Forward Time: 1.4401s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0028 | max_forward: 0.0602 | max_loss: 0.0029 | max_backward: 0.1260 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.8488 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8190.6 | uss_MB: 7045.2 | child_rss_MB: 23908.5 | allocated_MB: 198.3 | reserved_MB: 22482.0 | inactive_split_MB: 383.7 +rank0 [2026-11-15 05:18:33,323] INFO : Running validation on rank0 +rank0 [2026-11-15 05:18:33,324] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:18:33,324] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:18:33,948] INFO : saved batch0_epoch39000.parquet +rank0 [2026-11-15 05:18:34,070] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:18:34,185] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:18:34,255] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:18:34,323] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:18:34,389] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:18:34,457] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:18:34,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:18:34,636] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:18:34,742] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:18:34,845] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:18:34,912] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:18:34,978] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:18:35,045] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:18:35,112] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:18:35,179] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:18:35,281] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:18:35,288] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:18:35,291] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:18:35,293] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:18:35,296] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:18:35,299] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:18:35,301] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:18:35,303] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:18:35,306] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:18:35,308] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:18:35,311] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:18:35,313] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:18:35,316] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:18:35,318] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:18:35,321] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:18:35,323] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:18:43,071] INFO : VALIDATION | Step=39000/500000 | Train Loss=2.1727 | Valid Loss=2.5693 | Train Loss Unweighted=0.9065 | Valid Loss Unweighted=0.8747 | Stale=0 | ETA=2125.1m +rank0 [2026-11-15 05:18:43,071] INFO : split_configs=['1'] +rank0 [2026-11-15 05:18:43,082] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:18:43,090] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:18:43,189] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:18:44,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:18:44,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:18:44,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:18:45,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:18:45,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:18:45,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:18:46,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:18:46,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:18:46,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:18:47,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:18:47,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:18:47,890] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:18:48,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:18:48,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:18:48,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:18:49,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:18:49,246] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:18:49,248] INFO : split_configs=['1'] +rank0 [2026-11-15 05:18:49,259] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:18:49,263] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:18:49,328] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:18:50,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:18:50,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:18:51,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:18:51,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:18:52,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:18:52,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:18:53,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:18:53,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:18:53,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:18:54,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:18:54,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:18:55,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:18:55,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:18:56,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:18:56,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:18:57,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:18:57,157] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:18:57,160] INFO : split_configs=['1'] +rank0 [2026-11-15 05:18:57,172] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:18:57,175] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:18:57,243] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:18:58,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:18:58,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:18:59,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:18:59,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:18:59,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:19:00,382] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:19:00,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:19:01,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:19:01,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:19:01,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:19:02,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:19:02,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:19:03,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:19:03,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:19:04,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:19:04,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:19:04,548] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:19:04,554] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf +rank0 [2026-11-15 05:19:05,729] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf +rank0 [2026-11-15 05:19:06,153] INFO : Plotted number of elements +rank0 [2026-11-15 05:19:07,445] INFO : Plotted jets +rank0 [2026-11-15 05:19:08,931] INFO : Plotted jet ratio +rank0 [2026-11-15 05:19:08,942] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:19:10,614] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:19:11,057] INFO : Plotted number of elements +rank0 [2026-11-15 05:19:12,391] INFO : Plotted jets +rank0 [2026-11-15 05:19:13,420] INFO : Plotted jet ratio +rank0 [2026-11-15 05:19:13,428] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:19:14,943] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:19:15,394] INFO : Plotted number of elements +rank0 [2026-11-15 05:19:16,704] INFO : Plotted jets +rank0 [2026-11-15 05:19:17,693] INFO : Plotted jet ratio +rank0 [2026-11-15 05:19:18,135] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 05:19:18,135] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:23:11,791] INFO : Step 40000/500000 rank0 | Train Loss: 2.6374 | Train Loss Unweighted: 0.8317 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2701 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2619 | Regression_eta: 0.0467 | Regression_pt: 0.2304 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1128 | Classification_binary: 0.2701 | Regression_cos_phi: 0.6592 | Regression_energy: 0.1568 | Regression_eta: 0.6061 | Regression_pt: 0.1378 | Regression_sin_phi: 0.6947 | LR: 9.94e-05 | DataLoad Time: 0.0023s | Model Forward Time: 1.5348s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0677 | max_loss: 0.0029 | max_backward: 0.1446 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.8439 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8259.3 | uss_MB: 7114.0 | child_rss_MB: 23641.0 | allocated_MB: 201.2 | reserved_MB: 16446.0 | inactive_split_MB: 560.8 +rank0 [2026-11-15 05:23:11,796] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-40000.pth +rank0 [2026-11-15 05:23:11,866] INFO : Running validation on rank0 +rank0 [2026-11-15 05:23:11,866] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:23:11,866] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:23:12,495] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-15 05:23:12,623] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:23:12,736] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:23:12,808] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:23:12,875] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:23:12,941] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:23:13,007] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:23:13,077] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:23:13,189] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:23:13,297] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:23:13,403] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:23:13,474] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:23:13,542] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:23:13,608] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:23:13,674] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:23:13,741] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:23:13,844] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:23:13,852] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:23:13,854] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:23:13,857] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:23:13,859] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:23:13,862] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:23:13,865] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:23:13,867] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:23:13,870] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:23:13,872] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:23:13,874] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:23:13,877] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:23:13,879] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:23:13,882] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:23:13,885] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:23:13,887] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:23:21,580] INFO : VALIDATION | Step=40000/500000 | Train Loss=2.6374 | Valid Loss=2.5945 | Train Loss Unweighted=0.8317 | Valid Loss Unweighted=0.8703 | Stale=1 | ETA=2120.8m +rank0 [2026-11-15 05:23:21,581] INFO : split_configs=['1'] +rank0 [2026-11-15 05:23:21,591] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:23:21,599] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:23:21,680] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:23:22,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:23:22,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:23:23,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:23:23,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:23:23,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:23:24,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:23:24,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:23:24,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:23:25,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:23:25,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:23:25,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:23:26,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:23:26,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:23:26,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:23:27,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:23:27,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:23:27,655] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 05:23:27,659] INFO : split_configs=['1'] +rank0 [2026-11-15 05:23:27,671] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:23:27,674] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:23:27,733] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:23:28,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:23:29,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:23:29,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:23:30,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:23:30,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:23:30,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:23:31,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:23:31,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:23:32,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:23:32,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:23:33,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:23:33,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:23:34,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:23:34,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:23:34,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:23:35,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:23:35,521] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:23:35,524] INFO : split_configs=['1'] +rank0 [2026-11-15 05:23:35,536] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:23:35,540] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:23:35,600] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:23:36,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:23:37,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:23:37,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:23:37,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:23:38,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:23:38,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:23:39,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:23:39,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:23:39,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:23:40,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:23:40,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:23:41,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:23:41,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:23:41,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:23:42,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:23:42,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:23:42,856] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:23:42,863] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf +rank0 [2026-11-15 05:23:44,035] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf +rank0 [2026-11-15 05:23:44,466] INFO : Plotted number of elements +rank0 [2026-11-15 05:23:45,741] INFO : Plotted jets +rank0 [2026-11-15 05:23:47,236] INFO : Plotted jet ratio +rank0 [2026-11-15 05:23:47,247] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:23:48,882] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:23:49,333] INFO : Plotted number of elements +rank0 [2026-11-15 05:23:50,667] INFO : Plotted jets +rank0 [2026-11-15 05:23:51,695] INFO : Plotted jet ratio +rank0 [2026-11-15 05:23:51,703] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:23:53,145] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:23:53,592] INFO : Plotted number of elements +rank0 [2026-11-15 05:23:54,904] INFO : Plotted jets +rank0 [2026-11-15 05:23:55,898] INFO : Plotted jet ratio +rank0 [2026-11-15 05:25:52,071] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 05:25:52,072] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:27:49,452] INFO : Step 41000/500000 rank0 | Train Loss: 2.3469 | Train Loss Unweighted: 0.7935 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2603 | Regression_eta: 0.0263 | Regression_pt: 0.2458 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1359 | Classification_binary: 0.2387 | Regression_cos_phi: 0.6683 | Regression_energy: 0.1558 | Regression_eta: 0.3420 | Regression_pt: 0.1470 | Regression_sin_phi: 0.6593 | LR: 1.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5137s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0645 | max_loss: 0.0031 | max_backward: 0.1359 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.8821 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8438.9 | uss_MB: 7293.5 | child_rss_MB: 23753.6 | allocated_MB: 199.6 | reserved_MB: 21418.0 | inactive_split_MB: 220.4 +rank0 [2026-11-15 05:27:49,454] INFO : Running validation on rank0 +rank0 [2026-11-15 05:27:49,454] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:27:49,454] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:27:50,085] INFO : saved batch0_epoch41000.parquet +rank0 [2026-11-15 05:27:50,209] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:27:50,326] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:27:50,398] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:27:50,466] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:27:50,535] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:27:50,606] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:27:50,676] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:27:50,789] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:27:50,896] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:27:51,001] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:27:51,073] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:27:51,141] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:27:51,207] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:27:51,274] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:27:51,341] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:27:51,445] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:27:51,453] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:27:51,456] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:27:51,458] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:27:51,461] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:27:51,463] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:27:51,466] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:27:51,468] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:27:51,471] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:27:51,473] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:27:51,476] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:27:51,479] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:27:51,481] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:27:51,484] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:27:51,486] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:27:51,489] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:27:59,232] INFO : VALIDATION | Step=41000/500000 | Train Loss=2.3469 | Valid Loss=2.6093 | Train Loss Unweighted=0.7935 | Valid Loss Unweighted=0.8628 | Stale=2 | ETA=2116.4m +rank0 [2026-11-15 05:27:59,232] INFO : split_configs=['1'] +rank0 [2026-11-15 05:27:59,243] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:27:59,251] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:27:59,353] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:28:00,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:28:00,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:28:01,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:28:01,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:28:01,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:28:02,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:28:02,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:28:02,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:28:03,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:28:03,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:28:03,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:28:04,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:28:04,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:28:04,763] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:28:05,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:28:05,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:28:05,507] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:28:05,510] INFO : split_configs=['1'] +rank0 [2026-11-15 05:28:05,521] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:28:05,524] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:28:05,591] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:28:06,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:28:07,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:28:07,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:28:08,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:28:08,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:28:08,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:28:09,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:28:09,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:28:10,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:28:10,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:28:11,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:28:11,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:28:12,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:28:12,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:28:12,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:28:13,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:28:13,454] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:28:13,458] INFO : split_configs=['1'] +rank0 [2026-11-15 05:28:13,469] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:28:13,472] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:28:13,542] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:28:14,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:28:15,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:28:15,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:28:15,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:28:16,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:28:16,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:28:17,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:28:17,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:28:17,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:28:18,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:28:18,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:28:19,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:28:19,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:28:19,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:28:20,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:28:20,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:28:20,816] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 05:28:20,824] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf +rank0 [2026-11-15 05:28:22,002] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf +rank0 [2026-11-15 05:28:22,433] INFO : Plotted number of elements +rank0 [2026-11-15 05:28:23,706] INFO : Plotted jets +rank0 [2026-11-15 05:28:25,214] INFO : Plotted jet ratio +rank0 [2026-11-15 05:28:25,226] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:28:26,866] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:28:27,297] INFO : Plotted number of elements +rank0 [2026-11-15 05:28:28,659] INFO : Plotted jets +rank0 [2026-11-15 05:28:29,690] INFO : Plotted jet ratio +rank0 [2026-11-15 05:28:29,699] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:28:31,175] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:28:31,609] INFO : Plotted number of elements +rank0 [2026-11-15 05:28:32,930] INFO : Plotted jets +rank0 [2026-11-15 05:28:33,906] INFO : Plotted jet ratio +rank0 [2026-11-15 05:32:27,411] INFO : Step 42000/500000 rank0 | Train Loss: 2.2835 | Train Loss Unweighted: 0.8997 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2219 | Regression_cos_phi: 0.0112 | Regression_energy: 0.3066 | Regression_eta: 0.0434 | Regression_pt: 0.3102 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0912 | Classification_binary: 0.2219 | Regression_cos_phi: 0.6580 | Regression_energy: 0.1835 | Regression_eta: 0.5637 | Regression_pt: 0.1855 | Regression_sin_phi: 0.3797 | LR: 1.07e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4428s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0636 | max_loss: 0.0028 | max_backward: 0.1360 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.8302 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8444.7 | uss_MB: 7299.3 | child_rss_MB: 23724.6 | allocated_MB: 195.0 | reserved_MB: 17728.0 | inactive_split_MB: 169.0 +rank0 [2026-11-15 05:32:27,413] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-42000.pth +rank0 [2026-11-15 05:32:27,489] INFO : Running validation on rank0 +rank0 [2026-11-15 05:32:27,490] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:32:27,490] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:32:28,114] INFO : saved batch0_epoch42000.parquet +rank0 [2026-11-15 05:32:28,237] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:32:28,353] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:32:28,425] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:32:28,493] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:32:28,561] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:32:28,632] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:32:28,700] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:32:28,811] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:32:28,915] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:32:29,017] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:32:29,089] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:32:29,156] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:32:29,221] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:32:29,288] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:32:29,353] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:32:29,456] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:32:29,469] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:32:29,472] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:32:29,474] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:32:29,477] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:32:29,480] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:32:29,482] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:32:29,484] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:32:29,487] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:32:29,489] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:32:29,492] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:32:29,495] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:32:29,497] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:32:29,500] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:32:29,503] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:32:29,505] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:32:37,181] INFO : VALIDATION | Step=42000/500000 | Train Loss=2.2835 | Valid Loss=2.5962 | Train Loss Unweighted=0.8997 | Valid Loss Unweighted=0.8641 | Stale=3 | ETA=2112.0m +rank0 [2026-11-15 05:32:37,181] INFO : split_configs=['1'] +rank0 [2026-11-15 05:32:37,192] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:32:37,199] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:32:37,291] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:32:38,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:32:38,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:32:38,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:32:39,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:32:39,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:32:39,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:32:40,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:32:40,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:32:40,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:32:41,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:32:41,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:32:42,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:32:42,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:32:42,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:32:42,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:32:43,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:32:43,415] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:32:43,418] INFO : split_configs=['1'] +rank0 [2026-11-15 05:32:43,430] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:32:43,434] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:32:43,500] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:32:44,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:32:45,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:32:45,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:32:45,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:32:46,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:32:46,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:32:47,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:32:47,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:32:48,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:32:48,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:32:49,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:32:49,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:32:49,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:32:50,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:32:50,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:32:51,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:32:51,375] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:32:51,378] INFO : split_configs=['1'] +rank0 [2026-11-15 05:32:51,389] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:32:51,392] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:32:51,463] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:32:52,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:32:53,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:32:53,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:32:53,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:32:54,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:32:54,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:32:55,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:32:55,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:32:55,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:32:56,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:32:56,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:32:57,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:32:57,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:32:57,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:32:58,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:32:58,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:32:58,850] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:32:58,857] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf +rank0 [2026-11-15 05:33:00,030] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf +rank0 [2026-11-15 05:33:00,460] INFO : Plotted number of elements +rank0 [2026-11-15 05:33:01,772] INFO : Plotted jets +rank0 [2026-11-15 05:33:03,306] INFO : Plotted jet ratio +rank0 [2026-11-15 05:33:03,316] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:33:05,022] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:33:05,458] INFO : Plotted number of elements +rank0 [2026-11-15 05:33:06,806] INFO : Plotted jets +rank0 [2026-11-15 05:33:07,842] INFO : Plotted jet ratio +rank0 [2026-11-15 05:33:07,852] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:33:09,301] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:33:09,753] INFO : Plotted number of elements +rank0 [2026-11-15 05:33:11,078] INFO : Plotted jets +rank0 [2026-11-15 05:33:12,078] INFO : Plotted jet ratio +rank0 [2026-11-15 05:33:12,524] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 05:33:12,524] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:37:06,785] INFO : Step 43000/500000 rank0 | Train Loss: 2.4192 | Train Loss Unweighted: 0.9571 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2364 | Regression_cos_phi: 0.0123 | Regression_energy: 0.3280 | Regression_eta: 0.0325 | Regression_pt: 0.3391 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1114 | Classification_binary: 0.2364 | Regression_cos_phi: 0.7247 | Regression_energy: 0.1963 | Regression_eta: 0.4221 | Regression_pt: 0.2028 | Regression_sin_phi: 0.5255 | LR: 1.11e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4804s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0025 | max_forward: 0.0559 | max_loss: 0.0033 | max_backward: 0.1164 | max_optimizer_step: 0.0836 | max_tensorboard_logging: 0.8846 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6896.9 | uss_MB: 5751.6 | child_rss_MB: 23657.1 | allocated_MB: 195.4 | reserved_MB: 21536.0 | inactive_split_MB: 178.6 +rank0 [2026-11-15 05:37:06,787] INFO : Running validation on rank0 +rank0 [2026-11-15 05:37:06,788] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:37:06,788] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:37:07,419] INFO : saved batch0_epoch43000.parquet +rank0 [2026-11-15 05:37:07,548] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:37:07,662] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:37:07,733] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:37:07,800] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:37:07,868] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:37:07,936] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:37:08,004] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:37:08,118] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:37:08,225] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:37:08,331] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:37:08,401] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:37:08,468] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:37:08,533] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:37:08,600] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:37:08,667] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:37:08,771] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:37:08,778] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:37:08,779] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:37:08,783] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:37:08,784] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:37:08,788] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:37:08,789] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:37:08,790] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:37:08,792] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:37:08,793] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:37:08,793] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:37:08,795] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:37:08,796] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:37:08,797] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:37:08,798] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:37:08,813] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:37:16,529] INFO : VALIDATION | Step=43000/500000 | Train Loss=2.4192 | Valid Loss=2.6712 | Train Loss Unweighted=0.9571 | Valid Loss Unweighted=0.8719 | Stale=4 | ETA=2107.9m +rank0 [2026-11-15 05:37:16,529] INFO : split_configs=['1'] +rank0 [2026-11-15 05:37:16,540] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:37:16,548] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:37:16,638] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:37:17,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:37:17,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:37:18,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:37:18,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:37:18,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:37:19,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:37:19,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:37:19,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:37:20,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:37:20,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:37:20,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:37:21,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:37:21,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:37:21,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:37:22,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:37:22,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:37:22,711] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:37:22,714] INFO : split_configs=['1'] +rank0 [2026-11-15 05:37:22,726] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:37:22,735] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:37:22,798] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:37:23,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:37:24,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:37:24,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:37:25,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:37:25,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:37:26,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:37:26,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:37:26,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:37:27,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:37:27,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:37:28,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:37:28,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:37:29,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:37:29,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:37:30,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:37:30,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:37:30,615] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:37:30,618] INFO : split_configs=['1'] +rank0 [2026-11-15 05:37:30,629] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:37:30,632] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:37:30,695] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:37:31,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:37:32,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:37:32,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:37:32,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:37:33,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:37:33,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:37:34,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:37:34,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:37:34,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:37:35,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:37:35,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:37:36,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:37:36,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:37:37,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:37:37,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:37:37,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:37:37,945] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:37:37,952] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf +rank0 [2026-11-15 05:37:39,141] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf +rank0 [2026-11-15 05:37:39,586] INFO : Plotted number of elements +rank0 [2026-11-15 05:37:40,887] INFO : Plotted jets +rank0 [2026-11-15 05:37:42,424] INFO : Plotted jet ratio +rank0 [2026-11-15 05:37:42,436] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:37:44,089] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:37:44,523] INFO : Plotted number of elements +rank0 [2026-11-15 05:37:45,874] INFO : Plotted jets +rank0 [2026-11-15 05:37:46,901] INFO : Plotted jet ratio +rank0 [2026-11-15 05:37:46,910] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:37:48,337] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:37:48,797] INFO : Plotted number of elements +rank0 [2026-11-15 05:37:50,118] INFO : Plotted jets +rank0 [2026-11-15 05:37:51,107] INFO : Plotted jet ratio +rank0 [2026-11-15 05:39:46,875] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 05:39:46,875] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:41:43,798] INFO : Step 44000/500000 rank0 | Train Loss: 2.5401 | Train Loss Unweighted: 0.9737 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2272 | Regression_cos_phi: 0.0074 | Regression_energy: 0.3460 | Regression_eta: 0.0824 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0906 | Classification_binary: 0.2272 | Regression_cos_phi: 0.4374 | Regression_energy: 0.2070 | Regression_eta: 1.0707 | Regression_pt: 0.1825 | Regression_sin_phi: 0.3244 | LR: 1.15e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4402s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0561 | max_loss: 0.0028 | max_backward: 0.1178 | max_optimizer_step: 0.0824 | max_tensorboard_logging: 0.8377 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8643.8 | uss_MB: 7498.5 | child_rss_MB: 23772.2 | allocated_MB: 194.6 | reserved_MB: 17098.0 | inactive_split_MB: 273.4 +rank0 [2026-11-15 05:41:43,800] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-44000.pth +rank0 [2026-11-15 05:41:43,867] INFO : Running validation on rank0 +rank0 [2026-11-15 05:41:43,867] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:41:43,867] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:41:44,483] INFO : saved batch0_epoch44000.parquet +rank0 [2026-11-15 05:41:44,603] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:41:44,719] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:41:44,790] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:41:44,857] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:41:44,922] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:41:44,989] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:41:45,056] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:41:45,166] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:41:45,270] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:41:45,372] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:41:45,439] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:41:45,504] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:41:45,569] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:41:45,634] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:41:45,700] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:41:45,802] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:41:45,809] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:41:45,812] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:41:45,815] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:41:45,817] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:41:45,820] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:41:45,822] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:41:45,825] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:41:45,828] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:41:45,830] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:41:45,832] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:41:45,836] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:41:45,838] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:41:45,841] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:41:45,844] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:41:45,846] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:41:53,545] INFO : VALIDATION | Step=44000/500000 | Train Loss=2.5401 | Valid Loss=2.6206 | Train Loss Unweighted=0.9737 | Valid Loss Unweighted=0.8645 | Stale=5 | ETA=2103.3m +rank0 [2026-11-15 05:41:53,545] INFO : split_configs=['1'] +rank0 [2026-11-15 05:41:53,556] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:41:53,564] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:41:53,658] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:41:54,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:41:54,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:41:55,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:41:55,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:41:56,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:41:56,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:41:56,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:41:57,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:41:57,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:41:57,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:41:58,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:41:58,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:41:58,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:41:59,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:41:59,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:41:59,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:41:59,823] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:41:59,825] INFO : split_configs=['1'] +rank0 [2026-11-15 05:41:59,837] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:41:59,841] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:41:59,911] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:42:01,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:42:01,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:42:01,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:42:02,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:42:02,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:42:03,217] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:42:03,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:42:04,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:42:04,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:42:05,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:42:05,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:42:05,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:42:06,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:42:06,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:42:07,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:42:07,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:42:07,800] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:42:07,803] INFO : split_configs=['1'] +rank0 [2026-11-15 05:42:07,814] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:42:07,819] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:42:07,891] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:42:08,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:42:09,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:42:09,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:42:10,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:42:10,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:42:11,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:42:11,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:42:11,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:42:12,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:42:12,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:42:13,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:42:13,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:42:13,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:42:14,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:42:14,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:42:15,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:42:15,275] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:42:15,282] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf +rank0 [2026-11-15 05:42:16,441] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf +rank0 [2026-11-15 05:42:16,870] INFO : Plotted number of elements +rank0 [2026-11-15 05:42:18,167] INFO : Plotted jets +rank0 [2026-11-15 05:42:19,701] INFO : Plotted jet ratio +rank0 [2026-11-15 05:42:19,713] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:42:21,347] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:42:21,782] INFO : Plotted number of elements +rank0 [2026-11-15 05:42:23,144] INFO : Plotted jets +rank0 [2026-11-15 05:42:24,176] INFO : Plotted jet ratio +rank0 [2026-11-15 05:42:24,185] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:42:25,631] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:42:26,089] INFO : Plotted number of elements +rank0 [2026-11-15 05:42:27,416] INFO : Plotted jets +rank0 [2026-11-15 05:42:28,407] INFO : Plotted jet ratio +rank0 [2026-11-15 05:46:22,955] INFO : Step 45000/500000 rank0 | Train Loss: 2.0649 | Train Loss Unweighted: 0.7352 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2323 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2356 | Regression_eta: 0.0260 | Regression_pt: 0.2228 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1195 | Classification_binary: 0.2323 | Regression_cos_phi: 0.4968 | Regression_energy: 0.1410 | Regression_eta: 0.3377 | Regression_pt: 0.1332 | Regression_sin_phi: 0.6043 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4756s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0660 | max_loss: 0.0033 | max_backward: 0.1388 | max_optimizer_step: 0.0839 | max_tensorboard_logging: 0.8424 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 8646.7 | uss_MB: 7501.4 | child_rss_MB: 23760.3 | allocated_MB: 198.2 | reserved_MB: 22054.0 | inactive_split_MB: 161.8 +rank0 [2026-11-15 05:46:22,958] INFO : Running validation on rank0 +rank0 [2026-11-15 05:46:22,958] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:46:22,958] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:46:23,586] INFO : saved batch0_epoch45000.parquet +rank0 [2026-11-15 05:46:23,708] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:46:23,823] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:46:23,894] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:46:23,961] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:46:24,029] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:46:24,101] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:46:24,171] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:46:24,283] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:46:24,391] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:46:24,496] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:46:24,564] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:46:24,631] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:46:24,697] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:46:24,764] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:46:24,831] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:46:24,932] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:46:24,939] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:46:24,942] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:46:24,945] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:46:24,947] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:46:24,950] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:46:24,952] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:46:24,955] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:46:24,957] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:46:24,959] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:46:24,962] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:46:24,965] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:46:24,967] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:46:24,969] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:46:24,972] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:46:24,974] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:46:32,703] INFO : VALIDATION | Step=45000/500000 | Train Loss=2.0649 | Valid Loss=2.6115 | Train Loss Unweighted=0.7352 | Valid Loss Unweighted=0.8602 | Stale=6 | ETA=2099.1m +rank0 [2026-11-15 05:46:32,703] INFO : split_configs=['1'] +rank0 [2026-11-15 05:46:32,714] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:46:32,972] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:46:33,077] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:46:34,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:46:34,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:46:34,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:46:35,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:46:35,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:46:35,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:46:36,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:46:36,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:46:36,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:46:37,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:46:37,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:46:37,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:46:38,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:46:38,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:46:38,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:46:39,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:46:39,212] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:46:39,216] INFO : split_configs=['1'] +rank0 [2026-11-15 05:46:39,228] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:46:39,233] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:46:39,302] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:46:40,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:46:40,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:46:41,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:46:41,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:46:42,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:46:42,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:46:43,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:46:43,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:46:44,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:46:44,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:46:44,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:46:45,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:46:45,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:46:46,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:46:46,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:46:47,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:46:47,215] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:46:47,219] INFO : split_configs=['1'] +rank0 [2026-11-15 05:46:47,230] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:46:47,233] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:46:47,307] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:46:48,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:46:48,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:46:49,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:46:49,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:46:50,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:46:50,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:46:50,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:46:51,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:46:51,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:46:52,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:46:52,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:46:52,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:46:53,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:46:53,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:46:54,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:46:54,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:46:54,676] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:46:54,684] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf +rank0 [2026-11-15 05:46:55,861] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf +rank0 [2026-11-15 05:46:56,286] INFO : Plotted number of elements +rank0 [2026-11-15 05:46:57,564] INFO : Plotted jets +rank0 [2026-11-15 05:46:59,093] INFO : Plotted jet ratio +rank0 [2026-11-15 05:46:59,104] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:47:00,701] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:47:01,134] INFO : Plotted number of elements +rank0 [2026-11-15 05:47:02,489] INFO : Plotted jets +rank0 [2026-11-15 05:47:03,534] INFO : Plotted jet ratio +rank0 [2026-11-15 05:47:03,543] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:47:04,965] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:47:05,426] INFO : Plotted number of elements +rank0 [2026-11-15 05:47:06,741] INFO : Plotted jets +rank0 [2026-11-15 05:47:07,724] INFO : Plotted jet ratio +rank0 [2026-11-15 05:47:08,185] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 05:47:08,185] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:51:01,376] INFO : Step 46000/500000 rank0 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9634 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2460 | Regression_cos_phi: 0.0060 | Regression_energy: 0.3387 | Regression_eta: 0.0353 | Regression_pt: 0.3326 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1010 | Classification_binary: 0.2460 | Regression_cos_phi: 0.3511 | Regression_energy: 0.2027 | Regression_eta: 0.4588 | Regression_pt: 0.1989 | Regression_sin_phi: 0.2779 | LR: 1.23e-04 | DataLoad Time: 0.0181s | Model Forward Time: 1.4919s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0560 | max_loss: 0.0031 | max_backward: 0.1182 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.8906 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7136.7 | uss_MB: 5991.4 | child_rss_MB: 23800.4 | allocated_MB: 198.5 | reserved_MB: 19086.0 | inactive_split_MB: 161.5 +rank0 [2026-11-15 05:51:01,379] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-46000.pth +rank0 [2026-11-15 05:51:01,444] INFO : Running validation on rank0 +rank0 [2026-11-15 05:51:01,444] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:51:01,444] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:51:02,069] INFO : saved batch0_epoch46000.parquet +rank0 [2026-11-15 05:51:02,196] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:51:02,309] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:51:02,381] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:51:02,450] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:51:02,518] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:51:02,587] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:51:02,657] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:51:02,770] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:51:02,874] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:51:02,978] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:51:03,048] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:51:03,117] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:51:03,184] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:51:03,250] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:51:03,317] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:51:03,419] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:51:03,427] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:51:03,429] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:51:03,432] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:51:03,434] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:51:03,437] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:51:03,440] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:51:03,442] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:51:03,445] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:51:03,447] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:51:03,449] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:51:03,452] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:51:03,454] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:51:03,457] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:51:03,460] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:51:03,462] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:51:11,133] INFO : VALIDATION | Step=46000/500000 | Train Loss=1.8364 | Valid Loss=2.6188 | Train Loss Unweighted=0.9634 | Valid Loss Unweighted=0.8559 | Stale=7 | ETA=2094.8m +rank0 [2026-11-15 05:51:11,133] INFO : split_configs=['1'] +rank0 [2026-11-15 05:51:11,144] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:51:11,150] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:51:11,240] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:51:12,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:51:12,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:51:12,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:51:13,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:51:13,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:51:13,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:51:14,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:51:14,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:51:14,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:51:15,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:51:15,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:51:15,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:51:16,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:51:16,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:51:16,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:51:17,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:51:17,272] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:51:17,275] INFO : split_configs=['1'] +rank0 [2026-11-15 05:51:17,286] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:51:17,290] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:51:17,355] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:51:18,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:51:18,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:51:19,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:51:19,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:51:20,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:51:20,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:51:21,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:51:21,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:51:21,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:51:22,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:51:22,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:51:23,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:51:23,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:51:24,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:51:24,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:51:25,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:51:25,171] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:51:25,175] INFO : split_configs=['1'] +rank0 [2026-11-15 05:51:25,185] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:51:25,189] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:51:25,256] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:51:26,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:51:26,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:51:27,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:51:27,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:51:27,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:51:28,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:51:28,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:51:29,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:51:29,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:51:29,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:51:30,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:51:30,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:51:31,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:51:31,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:51:31,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:51:32,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:51:32,479] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 05:51:32,491] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf +rank0 [2026-11-15 05:51:33,667] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf +rank0 [2026-11-15 05:51:34,111] INFO : Plotted number of elements +rank0 [2026-11-15 05:51:35,404] INFO : Plotted jets +rank0 [2026-11-15 05:51:36,933] INFO : Plotted jet ratio +rank0 [2026-11-15 05:51:36,944] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:51:38,591] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:51:39,025] INFO : Plotted number of elements +rank0 [2026-11-15 05:51:40,372] INFO : Plotted jets +rank0 [2026-11-15 05:51:41,395] INFO : Plotted jet ratio +rank0 [2026-11-15 05:51:41,405] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:51:42,854] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:51:43,309] INFO : Plotted number of elements +rank0 [2026-11-15 05:51:44,615] INFO : Plotted jets +rank0 [2026-11-15 05:51:45,610] INFO : Plotted jet ratio +rank0 [2026-11-15 05:53:42,324] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 05:53:42,325] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:55:39,877] INFO : Step 47000/500000 rank0 | Train Loss: 1.5613 | Train Loss Unweighted: 0.8124 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2241 | Regression_cos_phi: 0.0048 | Regression_energy: 0.2810 | Regression_eta: 0.0290 | Regression_pt: 0.2690 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0926 | Classification_binary: 0.2241 | Regression_cos_phi: 0.2821 | Regression_energy: 0.1682 | Regression_eta: 0.3773 | Regression_pt: 0.1609 | Regression_sin_phi: 0.2561 | LR: 1.27e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4507s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0564 | max_loss: 0.0029 | max_backward: 0.1175 | max_optimizer_step: 0.0835 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8724.1 | uss_MB: 7578.8 | child_rss_MB: 23698.7 | allocated_MB: 194.2 | reserved_MB: 19220.0 | inactive_split_MB: 219.8 +rank0 [2026-11-15 05:55:39,881] INFO : Running validation on rank0 +rank0 [2026-11-15 05:55:39,881] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:55:39,881] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:55:40,504] INFO : saved batch0_epoch47000.parquet +rank0 [2026-11-15 05:55:40,629] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:55:40,748] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:55:40,820] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:55:40,889] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:55:40,957] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:55:41,025] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:55:41,095] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:55:41,204] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:55:41,309] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:55:41,411] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:55:41,480] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:55:41,547] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:55:41,613] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:55:41,678] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:55:41,744] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:55:41,848] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:55:41,855] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:55:41,858] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:55:41,860] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:55:41,863] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:55:41,866] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:55:41,868] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:55:41,871] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:55:41,873] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:55:41,876] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:55:41,878] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:55:41,881] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:55:41,883] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:55:41,885] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:55:41,888] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:55:41,890] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:55:49,563] INFO : VALIDATION | Step=47000/500000 | Train Loss=1.5613 | Valid Loss=2.5871 | Train Loss Unweighted=0.8124 | Valid Loss Unweighted=0.8493 | Stale=8 | ETA=2090.4m +rank0 [2026-11-15 05:55:49,563] INFO : split_configs=['1'] +rank0 [2026-11-15 05:55:49,575] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:55:49,582] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:55:49,682] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:55:50,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:55:50,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:55:51,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:55:51,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:55:52,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:55:52,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:55:52,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:55:53,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:55:53,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:55:53,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:55:54,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:55:54,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:55:54,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:55:55,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:55:55,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:55:55,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:55:55,824] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:55:55,828] INFO : split_configs=['1'] +rank0 [2026-11-15 05:55:55,839] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:55:55,842] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:55:55,912] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:55:57,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:55:57,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:55:57,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:55:58,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:55:58,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:55:59,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:55:59,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:56:00,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:56:00,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:56:01,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:56:01,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:56:01,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:56:02,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:56:02,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:56:03,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:56:03,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:56:03,811] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:56:03,814] INFO : split_configs=['1'] +rank0 [2026-11-15 05:56:03,825] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:56:03,829] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:56:03,902] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:56:04,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:56:05,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:56:05,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:56:06,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:56:06,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:56:07,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:56:07,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:56:07,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:56:08,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:56:08,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:56:09,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:56:09,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:56:09,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:56:10,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:56:10,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:56:11,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:56:11,284] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:56:11,292] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf +rank0 [2026-11-15 05:56:12,451] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf +rank0 [2026-11-15 05:56:12,876] INFO : Plotted number of elements +rank0 [2026-11-15 05:56:14,153] INFO : Plotted jets +rank0 [2026-11-15 05:56:15,667] INFO : Plotted jet ratio +rank0 [2026-11-15 05:56:15,678] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:56:17,311] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:56:17,749] INFO : Plotted number of elements +rank0 [2026-11-15 05:56:19,083] INFO : Plotted jets +rank0 [2026-11-15 05:56:20,133] INFO : Plotted jet ratio +rank0 [2026-11-15 05:56:20,142] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:56:21,579] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:56:22,027] INFO : Plotted number of elements +rank0 [2026-11-15 05:56:23,342] INFO : Plotted jets +rank0 [2026-11-15 05:56:24,329] INFO : Plotted jet ratio +rank0 [2026-11-15 06:00:17,589] INFO : Step 48000/500000 rank0 | Train Loss: 1.8338 | Train Loss Unweighted: 0.8450 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2306 | Regression_cos_phi: 0.0069 | Regression_energy: 0.2920 | Regression_eta: 0.0203 | Regression_pt: 0.2872 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.2306 | Regression_cos_phi: 0.4067 | Regression_energy: 0.1748 | Regression_eta: 0.2639 | Regression_pt: 0.1718 | Regression_sin_phi: 0.4784 | LR: 1.31e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4919s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0025 | max_forward: 0.0683 | max_loss: 0.0033 | max_backward: 0.1495 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.8395 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8810.6 | uss_MB: 7665.3 | child_rss_MB: 23852.1 | allocated_MB: 196.4 | reserved_MB: 18934.0 | inactive_split_MB: 153.6 +rank0 [2026-11-15 06:00:17,591] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-48000.pth +rank0 [2026-11-15 06:00:17,659] INFO : Running validation on rank0 +rank0 [2026-11-15 06:00:17,659] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:00:17,659] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:00:18,281] INFO : saved batch0_epoch48000.parquet +rank0 [2026-11-15 06:00:18,403] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:00:18,517] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:00:18,588] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:00:18,655] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:00:18,723] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:00:18,792] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:00:18,860] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:00:18,973] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:00:19,082] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:00:19,189] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:00:19,258] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:00:19,325] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:00:19,390] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:00:19,456] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:00:19,524] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:00:19,625] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:00:19,632] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:00:19,635] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:00:19,637] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:00:19,640] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:00:19,643] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:00:19,645] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:00:19,647] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:00:19,650] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:00:19,652] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:00:19,655] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:00:19,657] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:00:19,660] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:00:19,662] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:00:19,665] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:00:19,667] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:00:27,462] INFO : VALIDATION | Step=48000/500000 | Train Loss=1.8338 | Valid Loss=2.5264 | Train Loss Unweighted=0.8450 | Valid Loss Unweighted=0.8470 | Stale=0 | ETA=2086.0m +rank0 [2026-11-15 06:00:27,463] INFO : split_configs=['1'] +rank0 [2026-11-15 06:00:27,475] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:00:27,482] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:00:27,583] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:00:28,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:00:28,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:00:29,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:00:29,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:00:29,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:00:30,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:00:30,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:00:30,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:00:31,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:00:31,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:00:31,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:00:32,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:00:32,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:00:32,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:00:33,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:00:33,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:00:33,724] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:00:33,727] INFO : split_configs=['1'] +rank0 [2026-11-15 06:00:33,737] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:00:33,741] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:00:33,811] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:00:34,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:00:35,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:00:35,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:00:36,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:00:36,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:00:37,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:00:37,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:00:38,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:00:38,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:00:38,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:00:39,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:00:39,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:00:40,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:00:40,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:00:41,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:00:41,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:00:41,671] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:00:41,674] INFO : split_configs=['1'] +rank0 [2026-11-15 06:00:41,721] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:00:41,725] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:00:41,791] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:00:42,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:00:43,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:00:43,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:00:44,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:00:44,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:00:44,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:00:45,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:00:45,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:00:46,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:00:46,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:00:46,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:00:47,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:00:47,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:00:48,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:00:48,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:00:49,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:00:49,138] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:00:49,147] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf +rank0 [2026-11-15 06:00:50,353] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf +rank0 [2026-11-15 06:00:50,790] INFO : Plotted number of elements +rank0 [2026-11-15 06:00:52,076] INFO : Plotted jets +rank0 [2026-11-15 06:00:53,598] INFO : Plotted jet ratio +rank0 [2026-11-15 06:00:53,611] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:00:55,286] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:00:55,718] INFO : Plotted number of elements +rank0 [2026-11-15 06:00:57,066] INFO : Plotted jets +rank0 [2026-11-15 06:00:58,099] INFO : Plotted jet ratio +rank0 [2026-11-15 06:00:58,108] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:00:59,542] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:01:00,007] INFO : Plotted number of elements +rank0 [2026-11-15 06:01:01,319] INFO : Plotted jets +rank0 [2026-11-15 06:01:02,322] INFO : Plotted jet ratio +rank0 [2026-11-15 06:01:02,770] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 06:01:02,770] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:04:56,331] INFO : Step 49000/500000 rank0 | Train Loss: 2.5550 | Train Loss Unweighted: 0.8573 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2263 | Regression_cos_phi: 0.0082 | Regression_energy: 0.2902 | Regression_eta: 0.0781 | Regression_pt: 0.2476 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1039 | Classification_binary: 0.2263 | Regression_cos_phi: 0.4837 | Regression_energy: 0.1737 | Regression_eta: 1.0146 | Regression_pt: 0.1481 | Regression_sin_phi: 0.4048 | LR: 1.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3543s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0545 | max_loss: 0.0030 | max_backward: 0.1127 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.7458 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 8884.7 | uss_MB: 7739.4 | child_rss_MB: 23926.1 | allocated_MB: 193.4 | reserved_MB: 18790.0 | inactive_split_MB: 202.6 +rank0 [2026-11-15 06:04:56,333] INFO : Running validation on rank0 +rank0 [2026-11-15 06:04:56,333] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:04:56,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:04:56,969] INFO : saved batch0_epoch49000.parquet +rank0 [2026-11-15 06:04:57,092] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:04:57,206] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:04:57,275] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:04:57,342] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:04:57,408] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:04:57,476] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:04:57,544] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:04:57,656] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:04:57,764] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:04:57,870] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:04:57,939] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:04:58,005] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:04:58,072] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:04:58,140] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:04:58,207] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:04:58,309] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:04:58,317] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:04:58,319] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:04:58,322] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:04:58,324] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:04:58,327] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:04:58,330] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:04:58,333] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:04:58,336] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:04:58,339] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:04:58,341] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:04:58,345] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:04:58,347] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:04:58,349] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:04:58,352] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:04:58,354] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:05:06,066] INFO : VALIDATION | Step=49000/500000 | Train Loss=2.5550 | Valid Loss=2.6168 | Train Loss Unweighted=0.8573 | Valid Loss Unweighted=0.8523 | Stale=1 | ETA=2081.6m +rank0 [2026-11-15 06:05:06,067] INFO : split_configs=['1'] +rank0 [2026-11-15 06:05:06,076] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:05:06,084] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:05:06,184] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:05:07,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:05:07,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:05:07,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:05:08,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:05:08,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:05:08,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:05:09,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:05:09,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:05:09,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:05:10,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:05:10,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:05:10,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:05:11,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:05:11,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:05:11,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:05:12,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:05:12,294] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:05:12,296] INFO : split_configs=['1'] +rank0 [2026-11-15 06:05:12,308] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:05:12,312] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:05:12,383] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:05:13,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:05:13,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:05:14,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:05:14,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:05:15,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:05:15,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:05:16,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:05:16,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:05:17,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:05:17,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:05:17,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:05:18,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:05:18,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:05:19,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:05:19,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:05:20,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:05:20,203] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:05:20,206] INFO : split_configs=['1'] +rank0 [2026-11-15 06:05:20,217] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:05:20,220] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:05:20,294] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:05:21,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:05:21,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:05:22,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:05:22,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:05:23,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:05:23,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:05:23,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:05:24,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:05:24,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:05:25,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:05:25,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:05:25,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:05:26,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:05:26,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:05:27,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:05:27,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:05:27,634] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:05:27,641] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf +rank0 [2026-11-15 06:05:28,823] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf +rank0 [2026-11-15 06:05:29,250] INFO : Plotted number of elements +rank0 [2026-11-15 06:05:30,525] INFO : Plotted jets +rank0 [2026-11-15 06:05:32,044] INFO : Plotted jet ratio +rank0 [2026-11-15 06:05:32,055] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:05:33,707] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:05:34,150] INFO : Plotted number of elements +rank0 [2026-11-15 06:05:35,482] INFO : Plotted jets +rank0 [2026-11-15 06:05:36,508] INFO : Plotted jet ratio +rank0 [2026-11-15 06:05:36,517] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:05:37,941] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:05:38,384] INFO : Plotted number of elements +rank0 [2026-11-15 06:05:39,688] INFO : Plotted jets +rank0 [2026-11-15 06:05:40,673] INFO : Plotted jet ratio +rank0 [2026-11-15 06:07:36,887] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 06:07:36,887] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:09:34,221] INFO : Step 50000/500000 rank0 | Train Loss: 2.4711 | Train Loss Unweighted: 0.9205 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2691 | Regression_cos_phi: 0.0094 | Regression_energy: 0.3114 | Regression_eta: 0.0361 | Regression_pt: 0.2824 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2691 | Regression_cos_phi: 0.5515 | Regression_energy: 0.1864 | Regression_eta: 0.4689 | Regression_pt: 0.1689 | Regression_sin_phi: 0.7447 | LR: 1.40e-04 | DataLoad Time: 0.0172s | Model Forward Time: 1.4133s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0540 | max_loss: 0.0031 | max_backward: 0.1127 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.8029 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 9111.1 | uss_MB: 7966.9 | child_rss_MB: 23837.2 | allocated_MB: 194.3 | reserved_MB: 17004.0 | inactive_split_MB: 265.7 +rank0 [2026-11-15 06:09:34,224] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-50000.pth +rank0 [2026-11-15 06:09:34,290] INFO : Running validation on rank0 +rank0 [2026-11-15 06:09:34,290] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:09:34,290] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:09:34,912] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-15 06:09:35,036] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:09:35,154] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:09:35,226] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:09:35,295] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:09:35,363] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:09:35,434] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:09:35,505] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:09:35,618] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:09:35,726] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:09:35,832] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:09:35,901] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:09:35,968] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:09:36,036] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:09:36,106] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:09:36,174] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:09:36,280] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:09:36,287] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:09:36,290] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:09:36,293] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:09:36,295] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:09:36,298] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:09:36,300] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:09:36,303] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:09:36,306] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:09:36,308] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:09:36,310] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:09:36,313] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:09:36,315] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:09:36,324] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:09:36,327] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:09:36,329] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:09:44,064] INFO : VALIDATION | Step=50000/500000 | Train Loss=2.4711 | Valid Loss=2.5399 | Train Loss Unweighted=0.9205 | Valid Loss Unweighted=0.8491 | Stale=2 | ETA=2077.2m +rank0 [2026-11-15 06:09:44,065] INFO : split_configs=['1'] +rank0 [2026-11-15 06:09:44,074] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:09:44,080] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:09:44,182] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:09:45,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:09:45,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:09:45,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:09:46,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:09:46,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:09:46,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:09:47,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:09:47,539] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:09:47,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:09:48,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:09:48,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:09:48,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:09:49,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:09:49,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:09:49,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:09:50,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:09:50,293] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:09:50,296] INFO : split_configs=['1'] +rank0 [2026-11-15 06:09:50,308] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:09:50,323] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:09:50,393] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:09:51,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:09:51,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:09:52,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:09:52,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:09:53,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:09:53,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:09:54,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:09:54,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:09:55,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:09:55,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:09:55,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:09:56,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:09:56,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:09:57,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:09:57,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:09:58,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:09:58,291] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:09:58,294] INFO : split_configs=['1'] +rank0 [2026-11-15 06:09:58,306] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:09:58,309] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:09:58,384] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:09:59,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:09:59,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:10:00,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:10:00,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:10:01,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:10:01,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:10:01,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:10:02,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:10:02,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:10:03,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:10:03,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:10:03,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:10:04,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:10:04,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:10:05,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:10:05,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:10:05,723] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:10:05,730] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf +rank0 [2026-11-15 06:10:06,901] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf +rank0 [2026-11-15 06:10:07,324] INFO : Plotted number of elements +rank0 [2026-11-15 06:10:08,600] INFO : Plotted jets +rank0 [2026-11-15 06:10:10,115] INFO : Plotted jet ratio +rank0 [2026-11-15 06:10:10,126] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:10:11,751] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:10:12,182] INFO : Plotted number of elements +rank0 [2026-11-15 06:10:13,525] INFO : Plotted jets +rank0 [2026-11-15 06:10:14,542] INFO : Plotted jet ratio +rank0 [2026-11-15 06:10:14,551] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:10:16,010] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:10:16,469] INFO : Plotted number of elements +rank0 [2026-11-15 06:10:17,788] INFO : Plotted jets +rank0 [2026-11-15 06:10:18,770] INFO : Plotted jet ratio +rank0 [2026-11-15 06:14:15,108] INFO : Step 51000/500000 rank0 | Train Loss: 2.0990 | Train Loss Unweighted: 0.8173 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2388 | Regression_cos_phi: 0.0056 | Regression_energy: 0.2687 | Regression_eta: 0.0305 | Regression_pt: 0.2619 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0975 | Classification_binary: 0.2388 | Regression_cos_phi: 0.3279 | Regression_energy: 0.1608 | Regression_eta: 0.3956 | Regression_pt: 0.1566 | Regression_sin_phi: 0.7216 | LR: 1.44e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4773s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0022 | max_forward: 0.0509 | max_loss: 0.0027 | max_backward: 0.1086 | max_optimizer_step: 0.0875 | max_tensorboard_logging: 0.8651 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9187.4 | uss_MB: 8043.1 | child_rss_MB: 23808.6 | allocated_MB: 193.8 | reserved_MB: 21746.0 | inactive_split_MB: 168.2 +rank0 [2026-11-15 06:14:15,158] INFO : Running validation on rank0 +rank0 [2026-11-15 06:14:15,158] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:14:15,158] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:14:15,825] INFO : saved batch0_epoch51000.parquet +rank0 [2026-11-15 06:14:15,947] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:14:16,086] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:14:16,157] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:14:16,224] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:14:16,290] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:14:16,357] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:14:16,423] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:14:16,533] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:14:16,638] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:14:16,740] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:14:16,808] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:14:16,875] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:14:16,939] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:14:17,005] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:14:17,075] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:14:17,177] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:14:17,197] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:14:17,210] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:14:17,213] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:14:17,277] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:14:17,286] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:14:17,288] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:14:17,291] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:14:17,306] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:14:17,308] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:14:17,310] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:14:17,313] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:14:17,315] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:14:17,318] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:14:17,321] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:14:17,323] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:14:25,056] INFO : VALIDATION | Step=51000/500000 | Train Loss=2.0990 | Valid Loss=2.6327 | Train Loss Unweighted=0.8173 | Valid Loss Unweighted=0.8565 | Stale=3 | ETA=2073.1m +rank0 [2026-11-15 06:14:25,056] INFO : split_configs=['1'] +rank0 [2026-11-15 06:14:25,068] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:14:25,089] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:14:25,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:14:26,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:14:26,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:14:26,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:14:27,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:14:27,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:14:27,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:14:28,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:14:28,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:14:28,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:14:29,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:14:29,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:14:29,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:14:30,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:14:30,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:14:30,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:14:31,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:14:31,292] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:14:31,295] INFO : split_configs=['1'] +rank0 [2026-11-15 06:14:31,306] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:14:31,309] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:14:31,375] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:14:32,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:14:32,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:14:33,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:14:33,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:14:34,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:14:34,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:14:35,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:14:35,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:14:36,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:14:36,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:14:36,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:14:37,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:14:37,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:14:38,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:14:38,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:14:39,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:14:39,283] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:14:39,287] INFO : split_configs=['1'] +rank0 [2026-11-15 06:14:39,298] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:14:39,301] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:14:39,376] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:14:40,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:14:40,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:14:41,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:14:41,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:14:42,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:14:42,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:14:42,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:14:43,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:14:43,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:14:44,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:14:44,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:14:45,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:14:45,474] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:14:45,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:14:46,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:14:46,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:14:46,803] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:14:46,811] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf +rank0 [2026-11-15 06:14:48,006] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf +rank0 [2026-11-15 06:14:48,439] INFO : Plotted number of elements +rank0 [2026-11-15 06:14:49,731] INFO : Plotted jets +rank0 [2026-11-15 06:14:51,247] INFO : Plotted jet ratio +rank0 [2026-11-15 06:14:51,257] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:14:52,903] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:14:53,336] INFO : Plotted number of elements +rank0 [2026-11-15 06:14:54,678] INFO : Plotted jets +rank0 [2026-11-15 06:14:55,704] INFO : Plotted jet ratio +rank0 [2026-11-15 06:14:55,715] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:14:57,219] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:14:57,681] INFO : Plotted number of elements +rank0 [2026-11-15 06:14:58,993] INFO : Plotted jets +rank0 [2026-11-15 06:14:59,979] INFO : Plotted jet ratio +rank0 [2026-11-15 06:15:00,431] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 06:15:00,431] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:18:53,917] INFO : Step 52000/500000 rank0 | Train Loss: 2.1793 | Train Loss Unweighted: 0.7986 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2479 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2545 | Regression_eta: 0.0338 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.2479 | Regression_cos_phi: 0.5447 | Regression_energy: 0.1523 | Regression_eta: 0.4395 | Regression_pt: 0.1458 | Regression_sin_phi: 0.5615 | LR: 1.49e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4872s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0608 | max_loss: 0.0031 | max_backward: 0.1280 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.8486 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9116.0 | uss_MB: 7971.8 | child_rss_MB: 23839.6 | allocated_MB: 195.9 | reserved_MB: 20338.0 | inactive_split_MB: 178.1 +rank0 [2026-11-15 06:18:53,919] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-52000.pth +rank0 [2026-11-15 06:18:53,984] INFO : Running validation on rank0 +rank0 [2026-11-15 06:18:53,986] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:18:53,986] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:18:54,613] INFO : saved batch0_epoch52000.parquet +rank0 [2026-11-15 06:18:54,734] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:18:54,847] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:18:54,917] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:18:54,986] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:18:55,054] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:18:55,125] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:18:55,194] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:18:55,305] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:18:55,411] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:18:55,514] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:18:55,581] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:18:55,647] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:18:55,712] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:18:55,778] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:18:55,844] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:18:55,944] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:18:55,952] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:18:55,954] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:18:55,957] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:18:55,959] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:18:55,962] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:18:55,964] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:18:55,967] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:18:55,969] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:18:55,972] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:18:55,974] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:18:55,977] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:18:55,979] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:18:55,981] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:18:55,984] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:18:55,986] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:19:03,688] INFO : VALIDATION | Step=52000/500000 | Train Loss=2.1793 | Valid Loss=2.5885 | Train Loss Unweighted=0.7986 | Valid Loss Unweighted=0.8498 | Stale=4 | ETA=2068.7m +rank0 [2026-11-15 06:19:03,689] INFO : split_configs=['1'] +rank0 [2026-11-15 06:19:03,700] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:19:03,707] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:19:03,831] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:19:04,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:19:05,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:19:05,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:19:05,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:19:06,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:19:06,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:19:06,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:19:07,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:19:07,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:19:07,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:19:08,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:19:08,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:19:08,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:19:09,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:19:09,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:19:10,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:19:10,113] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:19:10,117] INFO : split_configs=['1'] +rank0 [2026-11-15 06:19:10,128] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:19:10,131] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:19:10,208] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:19:11,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:19:11,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:19:12,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:19:12,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:19:13,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:19:13,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:19:13,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:19:14,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:19:14,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:19:15,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:19:15,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:19:16,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:19:16,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:19:17,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:19:17,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:19:18,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:19:18,121] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:19:18,124] INFO : split_configs=['1'] +rank0 [2026-11-15 06:19:18,137] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:19:18,140] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:19:18,214] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:19:19,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:19:19,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:19:20,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:19:20,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:19:20,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:19:21,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:19:21,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:19:22,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:19:22,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:19:23,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:19:23,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:19:23,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:19:24,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:19:24,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:19:25,099] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:19:25,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:19:25,642] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:19:25,649] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf +rank0 [2026-11-15 06:19:26,821] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf +rank0 [2026-11-15 06:19:27,249] INFO : Plotted number of elements +rank0 [2026-11-15 06:19:28,540] INFO : Plotted jets +rank0 [2026-11-15 06:19:30,099] INFO : Plotted jet ratio +rank0 [2026-11-15 06:19:30,111] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:19:31,717] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:19:32,160] INFO : Plotted number of elements +rank0 [2026-11-15 06:19:33,507] INFO : Plotted jets +rank0 [2026-11-15 06:19:34,540] INFO : Plotted jet ratio +rank0 [2026-11-15 06:19:34,550] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:19:36,022] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:19:36,489] INFO : Plotted number of elements +rank0 [2026-11-15 06:19:37,806] INFO : Plotted jets +rank0 [2026-11-15 06:19:38,805] INFO : Plotted jet ratio +rank0 [2026-11-15 06:21:35,268] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 06:21:35,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:23:32,724] INFO : Step 53000/500000 rank0 | Train Loss: 1.6658 | Train Loss Unweighted: 0.7735 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2311 | Regression_cos_phi: 0.0072 | Regression_energy: 0.2533 | Regression_eta: 0.0253 | Regression_pt: 0.2518 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0999 | Classification_binary: 0.2311 | Regression_cos_phi: 0.4240 | Regression_energy: 0.1516 | Regression_eta: 0.3286 | Regression_pt: 0.1506 | Regression_sin_phi: 0.2801 | LR: 1.53e-04 | DataLoad Time: 0.0144s | Model Forward Time: 1.4999s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0632 | max_loss: 0.0029 | max_backward: 0.1300 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9177.3 | uss_MB: 8033.0 | child_rss_MB: 23700.9 | allocated_MB: 197.1 | reserved_MB: 19812.0 | inactive_split_MB: 230.9 +rank0 [2026-11-15 06:23:32,729] INFO : Running validation on rank0 +rank0 [2026-11-15 06:23:32,730] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:23:32,730] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:23:33,361] INFO : saved batch0_epoch53000.parquet +rank0 [2026-11-15 06:23:33,483] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:23:33,595] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:23:33,665] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:23:33,734] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:23:33,801] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:23:33,870] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:23:33,938] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:23:34,051] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:23:34,157] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:23:34,260] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:23:34,327] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:23:34,393] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:23:34,458] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:23:34,523] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:23:34,588] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:23:34,690] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:23:34,697] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:23:34,700] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:23:34,703] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:23:34,705] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:23:34,708] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:23:34,710] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:23:34,713] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:23:34,715] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:23:34,718] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:23:34,720] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:23:34,723] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:23:34,725] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:23:34,728] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:23:34,730] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:23:34,733] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:23:42,400] INFO : VALIDATION | Step=53000/500000 | Train Loss=1.6658 | Valid Loss=2.6057 | Train Loss Unweighted=0.7735 | Valid Loss Unweighted=0.8462 | Stale=5 | ETA=2064.4m +rank0 [2026-11-15 06:23:42,401] INFO : split_configs=['1'] +rank0 [2026-11-15 06:23:42,413] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:23:42,420] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:23:42,547] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:23:43,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:23:43,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:23:44,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:23:44,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:23:44,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:23:45,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:23:45,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:23:45,890] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:23:46,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:23:46,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:23:46,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:23:47,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:23:47,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:23:47,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:23:48,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:23:48,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:23:48,663] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:23:48,665] INFO : split_configs=['1'] +rank0 [2026-11-15 06:23:48,676] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:23:48,679] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:23:48,751] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:23:49,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:23:50,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:23:50,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:23:51,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:23:51,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:23:52,107] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:23:52,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:23:53,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:23:53,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:23:53,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:23:54,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:23:54,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:23:55,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:23:55,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:23:56,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:23:56,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:23:56,714] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:23:56,717] INFO : split_configs=['1'] +rank0 [2026-11-15 06:23:56,728] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:23:56,731] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:23:56,807] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:23:57,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:23:58,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:23:58,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:23:59,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:23:59,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:23:59,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:24:00,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:24:00,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:24:01,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:24:01,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:24:01,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:24:02,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:24:02,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:24:03,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:24:03,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:24:04,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:24:04,172] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:24:04,179] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf +rank0 [2026-11-15 06:24:05,363] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf +rank0 [2026-11-15 06:24:05,798] INFO : Plotted number of elements +rank0 [2026-11-15 06:24:07,073] INFO : Plotted jets +rank0 [2026-11-15 06:24:08,612] INFO : Plotted jet ratio +rank0 [2026-11-15 06:24:08,624] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:24:10,216] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:24:10,678] INFO : Plotted number of elements +rank0 [2026-11-15 06:24:12,047] INFO : Plotted jets +rank0 [2026-11-15 06:24:13,101] INFO : Plotted jet ratio +rank0 [2026-11-15 06:24:13,110] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:24:14,551] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:24:14,986] INFO : Plotted number of elements +rank0 [2026-11-15 06:24:16,300] INFO : Plotted jets +rank0 [2026-11-15 06:24:17,280] INFO : Plotted jet ratio +rank0 [2026-11-15 06:28:10,909] INFO : Step 54000/500000 rank0 | Train Loss: 1.9095 | Train Loss Unweighted: 0.7506 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2291 | Regression_cos_phi: 0.0071 | Regression_energy: 0.2409 | Regression_eta: 0.0155 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.2291 | Regression_cos_phi: 0.4191 | Regression_energy: 0.1442 | Regression_eta: 0.2019 | Regression_pt: 0.1476 | Regression_sin_phi: 0.6861 | LR: 1.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5580s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0722 | max_loss: 0.0028 | max_backward: 0.1554 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9339.3 | uss_MB: 8195.1 | child_rss_MB: 23782.2 | allocated_MB: 199.3 | reserved_MB: 19972.0 | inactive_split_MB: 314.7 +rank0 [2026-11-15 06:28:10,913] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-54000.pth +rank0 [2026-11-15 06:28:10,984] INFO : Running validation on rank0 +rank0 [2026-11-15 06:28:10,984] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:28:10,984] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:28:11,614] INFO : saved batch0_epoch54000.parquet +rank0 [2026-11-15 06:28:11,741] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:28:11,857] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:28:11,929] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:28:11,997] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:28:12,068] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:28:12,139] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:28:12,208] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:28:12,319] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:28:12,423] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:28:12,527] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:28:12,596] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:28:12,664] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:28:12,729] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:28:12,796] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:28:12,862] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:28:12,965] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:28:12,972] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:28:12,975] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:28:12,978] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:28:12,980] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:28:12,983] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:28:12,985] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:28:12,988] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:28:12,990] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:28:12,993] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:28:12,995] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:28:12,998] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:28:13,000] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:28:13,002] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:28:13,005] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:28:13,007] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:28:20,681] INFO : VALIDATION | Step=54000/500000 | Train Loss=1.9095 | Valid Loss=2.5306 | Train Loss Unweighted=0.7506 | Valid Loss Unweighted=0.8396 | Stale=6 | ETA=2059.9m +rank0 [2026-11-15 06:28:20,682] INFO : split_configs=['1'] +rank0 [2026-11-15 06:28:20,693] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:28:20,701] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:28:20,793] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:28:21,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:28:22,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:28:22,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:28:22,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:28:23,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:28:23,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:28:23,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:28:24,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:28:24,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:28:24,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:28:25,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:28:25,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:28:25,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:28:26,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:28:26,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:28:26,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:28:26,906] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:28:26,911] INFO : split_configs=['1'] +rank0 [2026-11-15 06:28:26,922] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:28:26,925] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:28:26,995] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:28:28,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:28:28,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:28:29,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:28:29,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:28:29,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:28:30,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:28:30,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:28:31,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:28:31,763] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:28:32,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:28:32,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:28:33,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:28:33,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:28:34,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:28:34,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:28:34,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:28:34,983] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:28:34,986] INFO : split_configs=['1'] +rank0 [2026-11-15 06:28:34,997] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:28:35,001] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:28:35,073] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:28:36,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:28:36,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:28:37,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:28:37,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:28:37,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:28:38,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:28:38,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:28:39,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:28:39,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:28:39,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:28:40,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:28:40,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:28:41,171] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:28:41,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:28:41,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:28:42,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:28:42,514] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:28:42,522] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf +rank0 [2026-11-15 06:28:44,197] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf +rank0 [2026-11-15 06:28:44,634] INFO : Plotted number of elements +rank0 [2026-11-15 06:28:45,931] INFO : Plotted jets +rank0 [2026-11-15 06:28:47,489] INFO : Plotted jet ratio +rank0 [2026-11-15 06:28:47,505] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:28:49,201] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:28:49,635] INFO : Plotted number of elements +rank0 [2026-11-15 06:28:50,990] INFO : Plotted jets +rank0 [2026-11-15 06:28:52,022] INFO : Plotted jet ratio +rank0 [2026-11-15 06:28:52,032] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:28:53,515] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:28:53,961] INFO : Plotted number of elements +rank0 [2026-11-15 06:28:55,289] INFO : Plotted jets +rank0 [2026-11-15 06:28:56,278] INFO : Plotted jet ratio +rank0 [2026-11-15 06:28:56,746] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 06:28:56,746] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:32:50,384] INFO : Step 55000/500000 rank0 | Train Loss: 1.8708 | Train Loss Unweighted: 0.8813 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2422 | Regression_cos_phi: 0.0060 | Regression_energy: 0.3065 | Regression_eta: 0.0306 | Regression_pt: 0.2888 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0905 | Classification_binary: 0.2422 | Regression_cos_phi: 0.3545 | Regression_energy: 0.1834 | Regression_eta: 0.3980 | Regression_pt: 0.1727 | Regression_sin_phi: 0.4294 | LR: 1.62e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5283s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0582 | max_loss: 0.0032 | max_backward: 0.1229 | max_optimizer_step: 0.0893 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9460.4 | uss_MB: 8316.2 | child_rss_MB: 23820.0 | allocated_MB: 197.8 | reserved_MB: 18872.0 | inactive_split_MB: 202.2 +rank0 [2026-11-15 06:32:50,388] INFO : Running validation on rank0 +rank0 [2026-11-15 06:32:50,388] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:32:50,388] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:32:51,013] INFO : saved batch0_epoch55000.parquet +rank0 [2026-11-15 06:32:51,139] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:32:51,253] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:32:51,325] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:32:51,394] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:32:51,463] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:32:51,534] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:32:51,604] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:32:51,716] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:32:51,824] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:32:51,928] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:32:51,996] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:32:52,065] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:32:52,132] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:32:52,198] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:32:52,266] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:32:52,368] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:32:52,375] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:32:52,378] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:32:52,380] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:32:52,383] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:32:52,386] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:32:52,388] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:32:52,390] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:32:52,393] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:32:52,395] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:32:52,398] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:32:52,401] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:32:52,403] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:32:52,405] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:32:52,409] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:32:52,411] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:33:00,132] INFO : VALIDATION | Step=55000/500000 | Train Loss=1.8708 | Valid Loss=2.5842 | Train Loss Unweighted=0.8813 | Valid Loss Unweighted=0.8403 | Stale=7 | ETA=2055.6m +rank0 [2026-11-15 06:33:00,133] INFO : split_configs=['1'] +rank0 [2026-11-15 06:33:00,145] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:33:00,163] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:33:00,272] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:33:01,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:33:01,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:33:01,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:33:02,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:33:02,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:33:02,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:33:03,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:33:03,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:33:03,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:33:04,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:33:04,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:33:05,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:33:05,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:33:05,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:33:05,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:33:06,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:33:06,403] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 06:33:06,406] INFO : split_configs=['1'] +rank0 [2026-11-15 06:33:06,417] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:33:06,422] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:33:06,498] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:33:07,656] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:33:08,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:33:08,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:33:08,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:33:09,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:33:09,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:33:10,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:33:10,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:33:11,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:33:11,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:33:12,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:33:12,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:33:12,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:33:13,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:33:13,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:33:14,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:33:14,369] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:33:14,373] INFO : split_configs=['1'] +rank0 [2026-11-15 06:33:14,384] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:33:14,388] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:33:14,465] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:33:15,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:33:16,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:33:16,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:33:16,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:33:17,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:33:17,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:33:18,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:33:18,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:33:18,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:33:19,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:33:19,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:33:20,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:33:20,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:33:20,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:33:21,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:33:21,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:33:21,900] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:33:21,906] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf +rank0 [2026-11-15 06:33:23,066] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf +rank0 [2026-11-15 06:33:23,498] INFO : Plotted number of elements +rank0 [2026-11-15 06:33:24,789] INFO : Plotted jets +rank0 [2026-11-15 06:33:26,356] INFO : Plotted jet ratio +rank0 [2026-11-15 06:33:26,368] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:33:28,000] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:33:28,444] INFO : Plotted number of elements +rank0 [2026-11-15 06:33:29,798] INFO : Plotted jets +rank0 [2026-11-15 06:33:30,847] INFO : Plotted jet ratio +rank0 [2026-11-15 06:33:30,856] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:33:32,311] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:33:32,770] INFO : Plotted number of elements +rank0 [2026-11-15 06:33:34,094] INFO : Plotted jets +rank0 [2026-11-15 06:33:35,101] INFO : Plotted jet ratio +rank0 [2026-11-15 06:35:31,897] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 06:35:31,898] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:37:29,890] INFO : Step 56000/500000 rank0 | Train Loss: 2.1666 | Train Loss Unweighted: 0.9249 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2473 | Regression_cos_phi: 0.0061 | Regression_energy: 0.3136 | Regression_eta: 0.0291 | Regression_pt: 0.3170 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2473 | Regression_cos_phi: 0.3573 | Regression_energy: 0.1877 | Regression_eta: 0.3782 | Regression_pt: 0.1896 | Regression_sin_phi: 0.7318 | LR: 1.67e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5137s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0026 | max_forward: 0.0598 | max_loss: 0.0029 | max_backward: 0.1263 | max_optimizer_step: 0.0878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9460.5 | uss_MB: 8316.2 | child_rss_MB: 23703.6 | allocated_MB: 197.0 | reserved_MB: 22324.0 | inactive_split_MB: 209.0 +rank0 [2026-11-15 06:37:29,892] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-56000.pth +rank0 [2026-11-15 06:37:29,960] INFO : Running validation on rank0 +rank0 [2026-11-15 06:37:29,960] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:37:29,960] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:37:30,583] INFO : saved batch0_epoch56000.parquet +rank0 [2026-11-15 06:37:30,706] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:37:30,823] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:37:30,896] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:37:30,965] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:37:31,033] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:37:31,102] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:37:31,170] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:37:31,281] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:37:31,385] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:37:31,488] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:37:31,556] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:37:31,623] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:37:31,689] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:37:31,755] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:37:31,822] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:37:31,925] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:37:31,932] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:37:31,935] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:37:31,938] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:37:31,941] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:37:31,944] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:37:31,946] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:37:31,949] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:37:31,951] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:37:31,954] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:37:31,982] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:37:31,984] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:37:31,986] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:37:31,987] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:37:31,988] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:37:31,989] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:37:39,735] INFO : VALIDATION | Step=56000/500000 | Train Loss=2.1666 | Valid Loss=2.5654 | Train Loss Unweighted=0.9249 | Valid Loss Unweighted=0.8452 | Stale=8 | ETA=2051.3m +rank0 [2026-11-15 06:37:39,736] INFO : split_configs=['1'] +rank0 [2026-11-15 06:37:39,748] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:37:39,759] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:37:39,869] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:37:40,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:37:41,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:37:41,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:37:41,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:37:42,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:37:42,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:37:42,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:37:43,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:37:43,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:37:43,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:37:44,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:37:44,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:37:44,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:37:45,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:37:45,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:37:45,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:37:46,083] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:37:46,086] INFO : split_configs=['1'] +rank0 [2026-11-15 06:37:46,097] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:37:46,100] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:37:46,176] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:37:47,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:37:47,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:37:48,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:37:48,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:37:49,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:37:49,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:37:49,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:37:50,429] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:37:50,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:37:51,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:37:51,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:37:52,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:37:52,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:37:53,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:37:53,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:37:54,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:37:54,175] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:37:54,179] INFO : split_configs=['1'] +rank0 [2026-11-15 06:37:54,191] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:37:54,194] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:37:54,271] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:37:55,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:37:55,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:37:56,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:37:56,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:37:57,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:37:57,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:37:57,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:37:58,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:37:58,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:37:59,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:37:59,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:37:59,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:38:00,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:38:00,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:38:01,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:38:01,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:38:01,648] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:38:01,657] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf +rank0 [2026-11-15 06:38:02,814] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf +rank0 [2026-11-15 06:38:03,237] INFO : Plotted number of elements +rank0 [2026-11-15 06:38:04,532] INFO : Plotted jets +rank0 [2026-11-15 06:38:06,108] INFO : Plotted jet ratio +rank0 [2026-11-15 06:38:06,119] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:38:07,714] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:38:08,154] INFO : Plotted number of elements +rank0 [2026-11-15 06:38:09,509] INFO : Plotted jets +rank0 [2026-11-15 06:38:10,536] INFO : Plotted jet ratio +rank0 [2026-11-15 06:38:10,547] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:38:11,981] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:38:12,429] INFO : Plotted number of elements +rank0 [2026-11-15 06:38:13,754] INFO : Plotted jets +rank0 [2026-11-15 06:38:14,743] INFO : Plotted jet ratio +rank0 [2026-11-15 06:42:08,250] INFO : Step 57000/500000 rank0 | Train Loss: 1.5645 | Train Loss Unweighted: 0.7050 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2324 | Regression_cos_phi: 0.0084 | Regression_energy: 0.2249 | Regression_eta: 0.0161 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0948 | Classification_binary: 0.2324 | Regression_cos_phi: 0.4929 | Regression_energy: 0.1346 | Regression_eta: 0.2088 | Regression_pt: 0.1307 | Regression_sin_phi: 0.2702 | LR: 1.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7355s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0646 | max_loss: 0.0027 | max_backward: 0.1373 | max_optimizer_step: 0.0886 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9650.0 | uss_MB: 8505.8 | child_rss_MB: 23704.7 | allocated_MB: 200.7 | reserved_MB: 21426.0 | inactive_split_MB: 187.3 +rank0 [2026-11-15 06:42:08,253] INFO : Running validation on rank0 +rank0 [2026-11-15 06:42:08,254] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:42:08,254] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:42:08,889] INFO : saved batch0_epoch57000.parquet +rank0 [2026-11-15 06:42:09,015] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:42:09,132] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:42:09,204] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:42:09,272] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:42:09,338] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:42:09,405] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:42:09,471] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:42:09,583] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:42:09,691] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:42:09,797] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:42:09,868] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:42:09,937] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:42:10,006] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:42:10,077] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:42:10,146] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:42:10,253] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:42:10,260] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:42:10,263] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:42:10,265] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:42:10,268] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:42:10,271] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:42:10,273] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:42:10,275] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:42:10,278] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:42:10,280] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:42:10,283] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:42:10,285] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:42:10,288] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:42:10,290] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:42:10,293] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:42:10,295] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:42:17,968] INFO : VALIDATION | Step=57000/500000 | Train Loss=1.5645 | Valid Loss=2.5628 | Train Loss Unweighted=0.7050 | Valid Loss Unweighted=0.8422 | Stale=9 | ETA=2046.8m +rank0 [2026-11-15 06:42:17,969] INFO : split_configs=['1'] +rank0 [2026-11-15 06:42:17,979] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:42:17,987] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:42:18,087] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:42:19,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:42:19,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:42:19,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:42:20,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:42:20,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:42:20,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:42:21,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:42:21,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:42:21,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:42:22,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:42:22,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:42:22,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:42:23,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:42:23,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:42:23,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:42:24,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:42:24,176] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:42:24,181] INFO : split_configs=['1'] +rank0 [2026-11-15 06:42:24,192] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:42:24,195] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:42:24,265] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:42:25,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:42:25,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:42:26,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:42:26,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:42:27,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:42:27,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:42:28,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:42:28,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:42:28,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:42:29,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:42:29,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:42:30,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:42:30,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:42:31,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:42:31,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:42:32,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:42:32,164] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:42:32,167] INFO : split_configs=['1'] +rank0 [2026-11-15 06:42:32,179] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:42:32,182] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:42:32,254] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:42:33,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:42:33,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:42:34,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:42:34,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:42:35,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:42:35,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:42:35,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:42:36,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:42:36,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:42:37,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:42:37,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:42:37,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:42:38,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:42:38,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:42:39,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:42:39,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:42:39,642] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:42:39,649] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf +rank0 [2026-11-15 06:42:40,854] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf +rank0 [2026-11-15 06:42:41,289] INFO : Plotted number of elements +rank0 [2026-11-15 06:42:42,594] INFO : Plotted jets +rank0 [2026-11-15 06:42:44,146] INFO : Plotted jet ratio +rank0 [2026-11-15 06:42:44,157] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:42:45,843] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:42:46,281] INFO : Plotted number of elements +rank0 [2026-11-15 06:42:47,643] INFO : Plotted jets +rank0 [2026-11-15 06:42:48,676] INFO : Plotted jet ratio +rank0 [2026-11-15 06:42:48,685] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:42:50,113] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:42:50,580] INFO : Plotted number of elements +rank0 [2026-11-15 06:42:51,902] INFO : Plotted jets +rank0 [2026-11-15 06:42:52,892] INFO : Plotted jet ratio +rank0 [2026-11-15 06:42:53,364] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 06:42:53,364] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:46:47,590] INFO : Step 58000/500000 rank0 | Train Loss: 1.8553 | Train Loss Unweighted: 0.7971 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2124 | Regression_cos_phi: 0.0073 | Regression_energy: 0.2679 | Regression_eta: 0.0231 | Regression_pt: 0.2781 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0817 | Classification_binary: 0.2124 | Regression_cos_phi: 0.4296 | Regression_energy: 0.1603 | Regression_eta: 0.2995 | Regression_pt: 0.1663 | Regression_sin_phi: 0.5055 | LR: 1.76e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5340s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0633 | max_loss: 0.0030 | max_backward: 0.1339 | max_optimizer_step: 0.0897 | max_tensorboard_logging: 0.8589 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9597.2 | uss_MB: 8453.0 | child_rss_MB: 23815.2 | allocated_MB: 198.0 | reserved_MB: 19232.0 | inactive_split_MB: 648.0 +rank0 [2026-11-15 06:46:47,592] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-58000.pth +rank0 [2026-11-15 06:46:47,659] INFO : Running validation on rank0 +rank0 [2026-11-15 06:46:47,659] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:46:47,659] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:46:48,282] INFO : saved batch0_epoch58000.parquet +rank0 [2026-11-15 06:46:48,404] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:46:48,522] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:46:48,594] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:46:48,662] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:46:48,729] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:46:48,796] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:46:48,864] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:46:48,971] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:46:49,080] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:46:49,183] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:46:49,251] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:46:49,318] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:46:49,384] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:46:49,451] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:46:49,518] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:46:49,621] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:46:49,666] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:46:49,670] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:46:49,672] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:46:49,675] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:46:49,677] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:46:49,680] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:46:49,682] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:46:49,685] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:46:49,687] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:46:49,689] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:46:49,692] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:46:49,694] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:46:49,696] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:46:49,699] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:46:49,701] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:46:57,418] INFO : VALIDATION | Step=58000/500000 | Train Loss=1.8553 | Valid Loss=2.6198 | Train Loss Unweighted=0.7971 | Valid Loss Unweighted=0.8577 | Stale=10 | ETA=2042.5m +rank0 [2026-11-15 06:46:57,418] INFO : split_configs=['1'] +rank0 [2026-11-15 06:46:57,429] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:46:57,437] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:46:57,543] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:46:58,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:46:58,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:46:59,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:46:59,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:46:59,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:47:00,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:47:00,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:47:00,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:47:01,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:47:01,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:47:01,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:47:02,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:47:02,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:47:02,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:47:03,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:47:03,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:47:03,757] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:47:03,760] INFO : split_configs=['1'] +rank0 [2026-11-15 06:47:03,771] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:47:03,774] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:47:03,852] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:47:05,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:47:05,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:47:05,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:47:06,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:47:06,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:47:07,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:47:07,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:47:08,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:47:08,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:47:09,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:47:09,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:47:09,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:47:10,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:47:10,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:47:11,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:47:11,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:47:11,821] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:47:11,825] INFO : split_configs=['1'] +rank0 [2026-11-15 06:47:11,836] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:47:11,840] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:47:11,918] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:47:13,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:47:13,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:47:13,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:47:14,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:47:14,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:47:15,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:47:15,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:47:15,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:47:16,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:47:16,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:47:17,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:47:17,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:47:17,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:47:18,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:47:18,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:47:19,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:47:19,275] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:47:19,282] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf +rank0 [2026-11-15 06:47:20,497] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf +rank0 [2026-11-15 06:47:20,918] INFO : Plotted number of elements +rank0 [2026-11-15 06:47:22,205] INFO : Plotted jets +rank0 [2026-11-15 06:47:23,751] INFO : Plotted jet ratio +rank0 [2026-11-15 06:47:23,763] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:47:25,429] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:47:25,857] INFO : Plotted number of elements +rank0 [2026-11-15 06:47:27,207] INFO : Plotted jets +rank0 [2026-11-15 06:47:28,230] INFO : Plotted jet ratio +rank0 [2026-11-15 06:47:28,239] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:47:29,700] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:47:30,156] INFO : Plotted number of elements +rank0 [2026-11-15 06:47:31,471] INFO : Plotted jets +rank0 [2026-11-15 06:47:32,465] INFO : Plotted jet ratio +rank0 [2026-11-15 06:49:28,624] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 06:49:28,624] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:51:25,612] INFO : Step 59000/500000 rank0 | Train Loss: 1.6872 | Train Loss Unweighted: 0.7391 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2432 | Regression_cos_phi: 0.0062 | Regression_energy: 0.2427 | Regression_eta: 0.0205 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.2432 | Regression_cos_phi: 0.3649 | Regression_energy: 0.1452 | Regression_eta: 0.2664 | Regression_pt: 0.1309 | Regression_sin_phi: 0.4646 | LR: 1.81e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5419s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0643 | max_loss: 0.0029 | max_backward: 0.1362 | max_optimizer_step: 0.0886 | max_tensorboard_logging: 0.8727 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9525.7 | uss_MB: 8381.4 | child_rss_MB: 23912.1 | allocated_MB: 198.5 | reserved_MB: 19170.0 | inactive_split_MB: 399.5 +rank0 [2026-11-15 06:51:25,616] INFO : Running validation on rank0 +rank0 [2026-11-15 06:51:25,616] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:51:25,616] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:51:26,239] INFO : saved batch0_epoch59000.parquet +rank0 [2026-11-15 06:51:26,359] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:51:26,474] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:51:26,545] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:51:26,613] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:51:26,681] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:51:26,753] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:51:26,823] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:51:26,935] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:51:27,040] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:51:27,145] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:51:27,213] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:51:27,280] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:51:27,345] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:51:27,412] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:51:27,478] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:51:27,580] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:51:27,587] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:51:27,590] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:51:27,593] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:51:27,595] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:51:27,598] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:51:27,600] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:51:27,603] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:51:27,605] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:51:27,608] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:51:27,610] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:51:27,613] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:51:27,615] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:51:27,617] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:51:27,620] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:51:27,622] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:51:35,393] INFO : VALIDATION | Step=59000/500000 | Train Loss=1.6872 | Valid Loss=2.5204 | Train Loss Unweighted=0.7391 | Valid Loss Unweighted=0.8354 | Stale=0 | ETA=2037.9m +rank0 [2026-11-15 06:51:35,393] INFO : split_configs=['1'] +rank0 [2026-11-15 06:51:35,414] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:51:35,422] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:51:35,531] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:51:36,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:51:36,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:51:37,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:51:37,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:51:37,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:51:38,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:51:38,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:51:38,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:51:39,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:51:39,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:51:39,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:51:40,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:51:40,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:51:40,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:51:41,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:51:41,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:51:41,702] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:51:41,705] INFO : split_configs=['1'] +rank0 [2026-11-15 06:51:41,716] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:51:41,719] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:51:41,795] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:51:42,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:51:43,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:51:43,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:51:44,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:51:44,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:51:45,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:51:45,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:51:46,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:51:46,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:51:46,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:51:47,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:51:47,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:51:48,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:51:48,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:51:49,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:51:49,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:51:49,647] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:51:49,651] INFO : split_configs=['1'] +rank0 [2026-11-15 06:51:49,662] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:51:49,665] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:51:49,742] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:51:50,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:51:51,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:51:51,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:51:52,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:51:52,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:51:52,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:51:53,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:51:53,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:51:54,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:51:54,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:51:54,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:51:55,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:51:55,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:51:56,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:51:56,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:51:57,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:51:57,114] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:51:57,140] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf +rank0 [2026-11-15 06:51:58,312] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf +rank0 [2026-11-15 06:51:58,745] INFO : Plotted number of elements +rank0 [2026-11-15 06:52:00,025] INFO : Plotted jets +rank0 [2026-11-15 06:52:01,599] INFO : Plotted jet ratio +rank0 [2026-11-15 06:52:01,611] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:52:03,286] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:52:03,720] INFO : Plotted number of elements +rank0 [2026-11-15 06:52:05,078] INFO : Plotted jets +rank0 [2026-11-15 06:52:06,108] INFO : Plotted jet ratio +rank0 [2026-11-15 06:52:06,119] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:52:07,586] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:52:08,028] INFO : Plotted number of elements +rank0 [2026-11-15 06:52:09,363] INFO : Plotted jets +rank0 [2026-11-15 06:52:10,353] INFO : Plotted jet ratio +rank0 [2026-11-15 06:56:04,779] INFO : Step 60000/500000 rank0 | Train Loss: 1.8546 | Train Loss Unweighted: 0.7357 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2270 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2407 | Regression_eta: 0.0089 | Regression_pt: 0.2404 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1195 | Classification_binary: 0.2270 | Regression_cos_phi: 0.6235 | Regression_energy: 0.1440 | Regression_eta: 0.1153 | Regression_pt: 0.1438 | Regression_sin_phi: 0.4815 | LR: 1.86e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5127s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0550 | max_loss: 0.0028 | max_backward: 0.1164 | max_optimizer_step: 0.0892 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8148.9 | uss_MB: 7004.7 | child_rss_MB: 23923.1 | allocated_MB: 192.2 | reserved_MB: 20668.0 | inactive_split_MB: 151.8 +rank0 [2026-11-15 06:56:04,783] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-60000.pth +rank0 [2026-11-15 06:56:04,850] INFO : Running validation on rank0 +rank0 [2026-11-15 06:56:04,850] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:56:04,850] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:56:05,479] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-15 06:56:05,604] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:56:05,719] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:56:05,791] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:56:05,859] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:56:05,928] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:56:05,997] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:56:06,068] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:56:06,180] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:56:06,285] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:56:06,388] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:56:06,456] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:56:06,522] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:56:06,587] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:56:06,653] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:56:06,718] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:56:06,820] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:56:06,828] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:56:06,831] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:56:06,833] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:56:06,836] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:56:06,839] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:56:06,841] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:56:06,843] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:56:06,846] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:56:06,848] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:56:06,851] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:56:06,853] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:56:06,856] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:56:06,858] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:56:06,861] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:56:06,863] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:56:14,550] INFO : VALIDATION | Step=60000/500000 | Train Loss=1.8546 | Valid Loss=2.6250 | Train Loss Unweighted=0.7357 | Valid Loss Unweighted=0.8436 | Stale=1 | ETA=2033.6m +rank0 [2026-11-15 06:56:14,550] INFO : split_configs=['1'] +rank0 [2026-11-15 06:56:14,562] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:56:14,568] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:56:14,683] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:56:15,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:56:15,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:56:16,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:56:16,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:56:17,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:56:17,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:56:17,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:56:18,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:56:18,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:56:18,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:56:19,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:56:19,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:56:19,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:56:20,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:56:20,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:56:20,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:56:20,842] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:56:20,845] INFO : split_configs=['1'] +rank0 [2026-11-15 06:56:20,856] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:56:20,859] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:56:20,932] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:56:22,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:56:22,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:56:22,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:56:23,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:56:23,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:56:24,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:56:24,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:56:25,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:56:25,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:56:26,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:56:26,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:56:26,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:56:27,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:56:27,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:56:28,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:56:28,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:56:28,834] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:56:28,838] INFO : split_configs=['1'] +rank0 [2026-11-15 06:56:28,849] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:56:28,853] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:56:28,927] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:56:30,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:56:30,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:56:30,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:56:31,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:56:31,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:56:32,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:56:32,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:56:32,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:56:33,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:56:33,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:56:34,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:56:34,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:56:34,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:56:35,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:56:35,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:56:36,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:56:36,352] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:56:36,361] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf +rank0 [2026-11-15 06:56:37,587] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf +rank0 [2026-11-15 06:56:38,017] INFO : Plotted number of elements +rank0 [2026-11-15 06:56:39,302] INFO : Plotted jets +rank0 [2026-11-15 06:56:40,849] INFO : Plotted jet ratio +rank0 [2026-11-15 06:56:40,861] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:56:42,576] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:56:43,005] INFO : Plotted number of elements +rank0 [2026-11-15 06:56:44,356] INFO : Plotted jets +rank0 [2026-11-15 06:56:45,380] INFO : Plotted jet ratio +rank0 [2026-11-15 06:56:45,391] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:56:46,881] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:56:47,326] INFO : Plotted number of elements +rank0 [2026-11-15 06:56:48,616] INFO : Plotted jets +rank0 [2026-11-15 06:56:49,601] INFO : Plotted jet ratio +rank0 [2026-11-15 06:56:50,060] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 06:56:50,060] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:00:43,595] INFO : Step 61000/500000 rank0 | Train Loss: 1.5491 | Train Loss Unweighted: 0.7963 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2433 | Regression_cos_phi: 0.0063 | Regression_energy: 0.2723 | Regression_eta: 0.0131 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.2433 | Regression_cos_phi: 0.3716 | Regression_energy: 0.1630 | Regression_eta: 0.1704 | Regression_pt: 0.1525 | Regression_sin_phi: 0.3810 | LR: 1.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5466s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0625 | max_loss: 0.0029 | max_backward: 0.1327 | max_optimizer_step: 0.0911 | max_tensorboard_logging: 0.8605 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 9900.2 | uss_MB: 8756.0 | child_rss_MB: 23680.6 | allocated_MB: 197.6 | reserved_MB: 21298.0 | inactive_split_MB: 188.4 +rank0 [2026-11-15 07:00:43,598] INFO : Running validation on rank0 +rank0 [2026-11-15 07:00:43,598] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:00:43,598] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:00:44,225] INFO : saved batch0_epoch61000.parquet +rank0 [2026-11-15 07:00:44,347] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:00:44,461] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:00:44,530] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:00:44,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:00:44,663] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:00:44,729] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:00:44,795] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:00:44,905] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:00:45,010] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:00:45,115] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:00:45,183] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:00:45,248] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:00:45,313] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:00:45,379] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:00:45,446] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:00:45,549] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:00:45,568] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:00:45,581] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:00:45,584] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:00:45,594] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:00:45,607] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:00:45,610] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:00:45,620] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:00:45,625] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:00:45,627] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:00:45,637] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:00:45,650] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:00:45,651] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:00:45,690] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:00:45,692] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:00:45,693] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:00:53,353] INFO : VALIDATION | Step=61000/500000 | Train Loss=1.5491 | Valid Loss=2.5733 | Train Loss Unweighted=0.7963 | Valid Loss Unweighted=0.8379 | Stale=2 | ETA=2029.1m +rank0 [2026-11-15 07:00:53,353] INFO : split_configs=['1'] +rank0 [2026-11-15 07:00:53,365] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:00:53,373] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:00:53,483] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:00:54,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:00:54,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:00:55,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:00:55,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:00:55,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:00:56,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:00:56,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:00:56,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:00:57,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:00:57,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:00:57,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:00:58,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:00:58,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:00:58,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:00:59,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:00:59,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:00:59,679] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:00:59,682] INFO : split_configs=['1'] +rank0 [2026-11-15 07:00:59,693] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:00:59,696] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:00:59,774] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:01:01,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:01:01,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:01:01,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:01:02,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:01:02,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:01:03,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:01:03,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:01:04,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:01:04,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:01:04,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:01:05,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:01:05,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:01:06,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:01:06,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:01:07,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:01:07,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:01:07,761] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:01:07,764] INFO : split_configs=['1'] +rank0 [2026-11-15 07:01:07,775] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:01:07,779] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:01:07,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:01:08,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:01:09,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:01:09,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:01:10,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:01:10,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:01:11,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:01:11,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:01:11,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:01:12,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:01:12,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:01:13,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:01:13,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:01:13,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:01:14,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:01:14,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:01:15,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:01:15,229] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:01:15,237] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf +rank0 [2026-11-15 07:01:16,428] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf +rank0 [2026-11-15 07:01:16,843] INFO : Plotted number of elements +rank0 [2026-11-15 07:01:18,133] INFO : Plotted jets +rank0 [2026-11-15 07:01:19,683] INFO : Plotted jet ratio +rank0 [2026-11-15 07:01:19,693] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:01:21,383] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:01:21,811] INFO : Plotted number of elements +rank0 [2026-11-15 07:01:23,173] INFO : Plotted jets +rank0 [2026-11-15 07:01:24,206] INFO : Plotted jet ratio +rank0 [2026-11-15 07:01:24,214] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:01:25,739] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:01:26,188] INFO : Plotted number of elements +rank0 [2026-11-15 07:01:27,527] INFO : Plotted jets +rank0 [2026-11-15 07:01:28,514] INFO : Plotted jet ratio +rank0 [2026-11-15 07:03:25,087] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 07:03:25,088] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:05:22,067] INFO : Step 62000/500000 rank0 | Train Loss: 1.8277 | Train Loss Unweighted: 0.7590 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2553 | Regression_cos_phi: 0.0063 | Regression_energy: 0.2335 | Regression_eta: 0.0399 | Regression_pt: 0.2186 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0923 | Classification_binary: 0.2553 | Regression_cos_phi: 0.3714 | Regression_energy: 0.1398 | Regression_eta: 0.5179 | Regression_pt: 0.1307 | Regression_sin_phi: 0.3204 | LR: 1.95e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5212s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0631 | max_loss: 0.0028 | max_backward: 0.1336 | max_optimizer_step: 0.0921 | max_tensorboard_logging: 0.8407 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9894.2 | uss_MB: 8750.1 | child_rss_MB: 23770.0 | allocated_MB: 198.1 | reserved_MB: 19254.0 | inactive_split_MB: 361.9 +rank0 [2026-11-15 07:05:22,070] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-62000.pth +rank0 [2026-11-15 07:05:22,136] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 07:05:22,150] INFO : Running validation on rank0 +rank0 [2026-11-15 07:05:22,150] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:05:22,150] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:05:22,774] INFO : saved batch0_epoch62000.parquet +rank0 [2026-11-15 07:05:22,898] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:05:23,015] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:05:23,089] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:05:23,157] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:05:23,224] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:05:23,291] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:05:23,357] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:05:23,466] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:05:23,571] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:05:23,674] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:05:23,742] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:05:23,808] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:05:23,873] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:05:23,940] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:05:24,006] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:05:24,109] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:05:24,154] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:05:24,181] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:05:24,184] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:05:24,211] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:05:24,238] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:05:24,241] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:05:24,258] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:05:24,262] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:05:24,264] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:05:24,267] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:05:24,269] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:05:24,272] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:05:24,278] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:05:24,280] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:05:24,283] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:05:31,971] INFO : VALIDATION | Step=62000/500000 | Train Loss=1.8277 | Valid Loss=2.5907 | Train Loss Unweighted=0.7590 | Valid Loss Unweighted=0.8503 | Stale=3 | ETA=2024.6m +rank0 [2026-11-15 07:05:31,971] INFO : split_configs=['1'] +rank0 [2026-11-15 07:05:31,983] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:05:31,992] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:05:32,102] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:05:33,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:05:33,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:05:33,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:05:34,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:05:34,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:05:34,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:05:35,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:05:35,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:05:35,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:05:36,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:05:36,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:05:36,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:05:37,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:05:37,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:05:37,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:05:38,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:05:38,316] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:05:38,320] INFO : split_configs=['1'] +rank0 [2026-11-15 07:05:38,331] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:05:38,336] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:05:38,416] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:05:39,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:05:40,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:05:40,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:05:40,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:05:41,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:05:41,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:05:42,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:05:42,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:05:43,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:05:43,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:05:44,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:05:44,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:05:44,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:05:45,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:05:45,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:05:46,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:05:46,425] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:05:46,428] INFO : split_configs=['1'] +rank0 [2026-11-15 07:05:46,440] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:05:46,444] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:05:46,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:05:47,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:05:48,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:05:48,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:05:48,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:05:49,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:05:49,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:05:50,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:05:50,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:05:50,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:05:51,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:05:51,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:05:52,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:05:52,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:05:53,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:05:53,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:05:53,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:05:53,957] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:05:53,976] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf +rank0 [2026-11-15 07:05:55,193] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf +rank0 [2026-11-15 07:05:55,623] INFO : Plotted number of elements +rank0 [2026-11-15 07:05:56,943] INFO : Plotted jets +rank0 [2026-11-15 07:05:58,526] INFO : Plotted jet ratio +rank0 [2026-11-15 07:05:58,550] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:06:00,202] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:06:00,637] INFO : Plotted number of elements +rank0 [2026-11-15 07:06:01,995] INFO : Plotted jets +rank0 [2026-11-15 07:06:03,027] INFO : Plotted jet ratio +rank0 [2026-11-15 07:06:03,048] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:06:04,511] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:06:04,952] INFO : Plotted number of elements +rank0 [2026-11-15 07:06:06,284] INFO : Plotted jets +rank0 [2026-11-15 07:06:07,276] INFO : Plotted jet ratio +rank0 [2026-11-15 07:10:01,921] INFO : Step 63000/500000 rank0 | Train Loss: 2.2889 | Train Loss Unweighted: 0.8857 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2466 | Regression_cos_phi: 0.0058 | Regression_energy: 0.2948 | Regression_eta: 0.0423 | Regression_pt: 0.2848 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1127 | Classification_binary: 0.2466 | Regression_cos_phi: 0.3427 | Regression_energy: 0.1764 | Regression_eta: 0.5494 | Regression_pt: 0.1703 | Regression_sin_phi: 0.6906 | LR: 2.00e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5475s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0577 | max_loss: 0.0029 | max_backward: 0.1231 | max_optimizer_step: 0.0922 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8373.2 | uss_MB: 7229.0 | child_rss_MB: 23872.9 | allocated_MB: 193.2 | reserved_MB: 18586.0 | inactive_split_MB: 306.8 +rank0 [2026-11-15 07:10:01,923] INFO : Running validation on rank0 +rank0 [2026-11-15 07:10:01,924] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:10:01,924] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:10:02,561] INFO : saved batch0_epoch63000.parquet +rank0 [2026-11-15 07:10:02,688] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:10:02,805] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:10:02,876] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:10:02,944] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:10:03,012] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:10:03,084] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:10:03,155] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:10:03,268] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:10:03,378] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:10:03,485] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:10:03,556] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:10:03,624] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:10:03,691] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:10:03,758] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:10:03,825] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:10:03,930] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:10:03,937] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:10:03,940] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:10:03,943] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:10:03,946] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:10:03,949] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:10:03,951] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:10:03,953] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:10:03,956] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:10:03,958] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:10:03,961] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:10:03,963] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:10:03,966] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:10:03,968] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:10:03,971] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:10:03,973] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:10:11,732] INFO : VALIDATION | Step=63000/500000 | Train Loss=2.2889 | Valid Loss=2.6352 | Train Loss Unweighted=0.8857 | Valid Loss Unweighted=0.8513 | Stale=4 | ETA=2020.3m +rank0 [2026-11-15 07:10:11,732] INFO : split_configs=['1'] +rank0 [2026-11-15 07:10:11,744] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:10:11,751] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:10:11,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:10:12,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:10:13,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:10:13,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:10:13,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:10:14,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:10:14,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:10:14,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:10:15,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:10:15,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:10:15,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:10:16,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:10:16,647] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:10:16,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:10:17,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:10:17,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:10:17,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:10:18,069] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:10:18,073] INFO : split_configs=['1'] +rank0 [2026-11-15 07:10:18,085] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:10:18,088] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:10:18,159] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:10:19,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:10:19,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:10:20,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:10:20,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:10:21,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:10:21,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:10:21,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:10:22,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:10:22,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:10:23,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:10:23,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:10:24,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:10:24,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:10:25,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:10:25,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:10:25,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:10:26,086] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:10:26,090] INFO : split_configs=['1'] +rank0 [2026-11-15 07:10:26,101] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:10:26,105] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:10:26,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:10:27,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:10:27,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:10:28,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:10:28,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:10:28,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:10:29,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:10:29,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:10:30,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:10:30,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:10:30,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:10:31,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:10:31,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:10:32,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:10:32,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:10:33,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:10:33,479] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:10:33,587] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:10:33,595] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf +rank0 [2026-11-15 07:10:34,812] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf +rank0 [2026-11-15 07:10:35,245] INFO : Plotted number of elements +rank0 [2026-11-15 07:10:36,550] INFO : Plotted jets +rank0 [2026-11-15 07:10:37,569] INFO : Plotted jet ratio +rank0 [2026-11-15 07:10:37,579] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:10:39,782] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:10:40,226] INFO : Plotted number of elements +rank0 [2026-11-15 07:10:41,614] INFO : Plotted jets +rank0 [2026-11-15 07:10:42,667] INFO : Plotted jet ratio +rank0 [2026-11-15 07:10:42,677] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:10:44,144] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:10:44,582] INFO : Plotted number of elements +rank0 [2026-11-15 07:10:45,913] INFO : Plotted jets +rank0 [2026-11-15 07:10:46,899] INFO : Plotted jet ratio +rank0 [2026-11-15 07:10:47,369] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 07:10:47,370] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:14:40,533] INFO : Step 64000/500000 rank0 | Train Loss: 1.4805 | Train Loss Unweighted: 0.8263 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2328 | Regression_cos_phi: 0.0055 | Regression_energy: 0.2774 | Regression_eta: 0.0147 | Regression_pt: 0.2908 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0925 | Classification_binary: 0.2328 | Regression_cos_phi: 0.3218 | Regression_energy: 0.1660 | Regression_eta: 0.1908 | Regression_pt: 0.1739 | Regression_sin_phi: 0.3026 | LR: 2.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5421s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0608 | max_loss: 0.0031 | max_backward: 0.1269 | max_optimizer_step: 0.0908 | max_tensorboard_logging: 0.8653 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9960.6 | uss_MB: 8816.4 | child_rss_MB: 23802.8 | allocated_MB: 198.8 | reserved_MB: 19994.0 | inactive_split_MB: 469.2 +rank0 [2026-11-15 07:14:40,536] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-64000.pth +rank0 [2026-11-15 07:14:40,602] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 07:14:40,616] INFO : Running validation on rank0 +rank0 [2026-11-15 07:14:40,616] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:14:40,616] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:14:41,240] INFO : saved batch0_epoch64000.parquet +rank0 [2026-11-15 07:14:41,362] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:14:41,479] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:14:41,551] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:14:41,621] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:14:41,688] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:14:41,756] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:14:41,823] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:14:41,935] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:14:42,041] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:14:42,148] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:14:42,218] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:14:42,287] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:14:42,356] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:14:42,425] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:14:42,493] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:14:42,598] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:14:42,641] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:14:42,645] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:14:42,646] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:14:42,647] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:14:42,648] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:14:42,649] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:14:42,651] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:14:42,652] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:14:42,661] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:14:42,662] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:14:42,663] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:14:42,664] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:14:42,665] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:14:42,667] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:14:42,668] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:14:50,414] INFO : VALIDATION | Step=64000/500000 | Train Loss=1.4805 | Valid Loss=2.6004 | Train Loss Unweighted=0.8263 | Valid Loss Unweighted=0.8299 | Stale=5 | ETA=2015.8m +rank0 [2026-11-15 07:14:50,415] INFO : split_configs=['1'] +rank0 [2026-11-15 07:14:50,426] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:14:50,434] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:14:50,545] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:14:51,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:14:51,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:14:52,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:14:52,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:14:52,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:14:53,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:14:53,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:14:53,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:14:54,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:14:54,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:14:55,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:14:55,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:14:55,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:14:55,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:14:56,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:14:56,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:14:56,796] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:14:56,799] INFO : split_configs=['1'] +rank0 [2026-11-15 07:14:56,811] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:14:56,814] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:14:56,892] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:14:58,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:14:58,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:14:58,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:14:59,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:14:59,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:15:00,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:15:00,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:15:01,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:15:01,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:15:02,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:15:02,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:15:02,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:15:03,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:15:03,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:15:04,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:15:04,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:15:04,845] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:15:04,849] INFO : split_configs=['1'] +rank0 [2026-11-15 07:15:04,861] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:15:04,865] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:15:04,947] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:15:06,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:15:06,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:15:06,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:15:07,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:15:07,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:15:08,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:15:08,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:15:08,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:15:09,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:15:09,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:15:10,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:15:10,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:15:11,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:15:11,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:15:11,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:15:12,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:15:12,352] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:15:12,359] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf +rank0 [2026-11-15 07:15:13,554] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf +rank0 [2026-11-15 07:15:13,972] INFO : Plotted number of elements +rank0 [2026-11-15 07:15:15,269] INFO : Plotted jets +rank0 [2026-11-15 07:15:16,851] INFO : Plotted jet ratio +rank0 [2026-11-15 07:15:16,862] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:15:18,518] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:15:18,946] INFO : Plotted number of elements +rank0 [2026-11-15 07:15:20,330] INFO : Plotted jets +rank0 [2026-11-15 07:15:21,352] INFO : Plotted jet ratio +rank0 [2026-11-15 07:15:21,363] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:15:22,865] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:15:23,303] INFO : Plotted number of elements +rank0 [2026-11-15 07:15:24,633] INFO : Plotted jets +rank0 [2026-11-15 07:15:25,616] INFO : Plotted jet ratio +rank0 [2026-11-15 07:17:22,499] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 07:17:22,499] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:19:20,586] INFO : Step 65000/500000 rank0 | Train Loss: 1.9476 | Train Loss Unweighted: 0.8759 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0081 | Regression_energy: 0.3007 | Regression_eta: 0.0351 | Regression_pt: 0.2962 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0912 | Classification_binary: 0.2301 | Regression_cos_phi: 0.4734 | Regression_energy: 0.1799 | Regression_eta: 0.4564 | Regression_pt: 0.1772 | Regression_sin_phi: 0.3394 | LR: 2.10e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7422s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0605 | max_loss: 0.0030 | max_backward: 0.1287 | max_optimizer_step: 0.0914 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10036.1 | uss_MB: 8891.9 | child_rss_MB: 23884.6 | allocated_MB: 195.2 | reserved_MB: 22066.0 | inactive_split_MB: 294.8 +rank0 [2026-11-15 07:19:20,590] INFO : Running validation on rank0 +rank0 [2026-11-15 07:19:20,590] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:19:20,590] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:19:21,220] INFO : saved batch0_epoch65000.parquet +rank0 [2026-11-15 07:19:21,341] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:19:21,455] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:19:21,525] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:19:21,591] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:19:21,657] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:19:21,724] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:19:21,791] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:19:21,901] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:19:22,005] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:19:22,110] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:19:22,179] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:19:22,246] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:19:22,312] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:19:22,380] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:19:22,448] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:19:22,553] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:19:22,573] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:19:22,585] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:19:22,588] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:19:22,610] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:19:22,616] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:19:22,618] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:19:22,621] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:19:22,630] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:19:22,632] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:19:22,635] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:19:22,637] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:19:22,640] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:19:22,642] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:19:22,645] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:19:22,647] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:19:30,363] INFO : VALIDATION | Step=65000/500000 | Train Loss=1.9476 | Valid Loss=2.5314 | Train Loss Unweighted=0.8759 | Valid Loss Unweighted=0.8316 | Stale=6 | ETA=2011.5m +rank0 [2026-11-15 07:19:30,363] INFO : split_configs=['1'] +rank0 [2026-11-15 07:19:30,375] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:19:30,383] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:19:30,488] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:19:31,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:19:31,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:19:32,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:19:32,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:19:32,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:19:33,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:19:33,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:19:33,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:19:34,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:19:34,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:19:34,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:19:35,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:19:35,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:19:35,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:19:36,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:19:36,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:19:36,642] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:19:36,646] INFO : split_configs=['1'] +rank0 [2026-11-15 07:19:36,658] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:19:36,661] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:19:36,733] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:19:37,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:19:38,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:19:38,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:19:39,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:19:39,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:19:40,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:19:40,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:19:40,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:19:41,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:19:41,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:19:42,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:19:42,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:19:43,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:19:43,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:19:44,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:19:44,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:19:44,604] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:19:44,607] INFO : split_configs=['1'] +rank0 [2026-11-15 07:19:44,621] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:19:44,624] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:19:44,698] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:19:45,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:19:46,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:19:46,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:19:47,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:19:47,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:19:47,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:19:48,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:19:48,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:19:49,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:19:49,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:19:49,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:19:50,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:19:50,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:19:51,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:19:51,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:19:51,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:19:52,094] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:19:52,102] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf +rank0 [2026-11-15 07:19:53,317] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf +rank0 [2026-11-15 07:19:53,752] INFO : Plotted number of elements +rank0 [2026-11-15 07:19:55,065] INFO : Plotted jets +rank0 [2026-11-15 07:19:56,094] INFO : Plotted jet ratio +rank0 [2026-11-15 07:19:56,105] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:19:57,797] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:19:58,785] INFO : Plotted number of elements +rank0 [2026-11-15 07:20:00,161] INFO : Plotted jets +rank0 [2026-11-15 07:20:01,229] INFO : Plotted jet ratio +rank0 [2026-11-15 07:20:01,239] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:20:02,715] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:20:03,161] INFO : Plotted number of elements +rank0 [2026-11-15 07:20:04,500] INFO : Plotted jets +rank0 [2026-11-15 07:20:05,501] INFO : Plotted jet ratio +rank0 [2026-11-15 07:23:59,569] INFO : Step 66000/500000 rank0 | Train Loss: 1.9954 | Train Loss Unweighted: 0.8365 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2530 | Regression_cos_phi: 0.0072 | Regression_energy: 0.2808 | Regression_eta: 0.0132 | Regression_pt: 0.2706 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0982 | Classification_binary: 0.2530 | Regression_cos_phi: 0.4205 | Regression_energy: 0.1680 | Regression_eta: 0.1713 | Regression_pt: 0.1618 | Regression_sin_phi: 0.7226 | LR: 2.15e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7803s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0614 | max_loss: 0.0031 | max_backward: 0.1299 | max_optimizer_step: 0.0922 | max_tensorboard_logging: 1.0998 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10090.1 | uss_MB: 8946.0 | child_rss_MB: 23639.5 | allocated_MB: 196.8 | reserved_MB: 15542.0 | inactive_split_MB: 209.2 +rank0 [2026-11-15 07:23:59,573] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-66000.pth +rank0 [2026-11-15 07:23:59,651] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 07:23:59,665] INFO : Running validation on rank0 +rank0 [2026-11-15 07:23:59,665] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:23:59,665] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:24:00,291] INFO : saved batch0_epoch66000.parquet +rank0 [2026-11-15 07:24:00,412] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:24:00,530] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:24:00,602] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:24:00,670] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:24:00,737] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:24:00,804] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:24:00,872] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:24:00,981] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:24:01,086] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:24:01,189] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:24:01,258] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:24:01,325] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:24:01,391] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:24:01,458] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:24:01,524] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:24:01,626] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:24:01,634] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:24:01,637] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:24:01,639] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:24:01,642] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:24:01,644] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:24:01,647] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:24:01,649] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:24:01,652] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:24:01,654] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:24:01,656] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:24:01,659] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:24:01,661] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:24:01,676] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:24:01,679] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:24:01,681] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:24:09,413] INFO : VALIDATION | Step=66000/500000 | Train Loss=1.9954 | Valid Loss=2.6272 | Train Loss Unweighted=0.8365 | Valid Loss Unweighted=0.8444 | Stale=7 | ETA=2007.0m +rank0 [2026-11-15 07:24:09,414] INFO : split_configs=['1'] +rank0 [2026-11-15 07:24:09,424] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:24:09,431] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:24:09,539] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:24:10,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:24:10,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:24:11,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:24:11,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:24:11,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:24:12,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:24:12,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:24:12,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:24:13,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:24:13,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:24:14,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:24:14,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:24:14,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:24:15,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:24:15,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:24:15,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:24:15,825] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:24:15,828] INFO : split_configs=['1'] +rank0 [2026-11-15 07:24:15,838] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:24:15,842] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:24:15,922] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:24:17,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:24:17,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:24:18,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:24:18,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:24:18,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:24:19,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:24:19,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:24:20,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:24:20,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:24:21,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:24:21,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:24:22,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:24:22,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:24:22,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:24:23,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:24:23,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:24:23,916] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:24:23,919] INFO : split_configs=['1'] +rank0 [2026-11-15 07:24:23,932] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:24:23,936] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:24:24,017] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:24:25,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:24:25,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:24:26,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:24:26,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:24:26,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:24:27,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:24:27,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:24:28,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:24:28,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:24:28,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:24:29,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:24:29,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:24:30,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:24:30,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:24:30,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:24:31,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:24:31,486] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:24:31,496] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf +rank0 [2026-11-15 07:24:32,713] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf +rank0 [2026-11-15 07:24:33,130] INFO : Plotted number of elements +rank0 [2026-11-15 07:24:34,409] INFO : Plotted jets +rank0 [2026-11-15 07:24:35,425] INFO : Plotted jet ratio +rank0 [2026-11-15 07:24:35,436] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:24:37,652] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:24:38,081] INFO : Plotted number of elements +rank0 [2026-11-15 07:24:39,431] INFO : Plotted jets +rank0 [2026-11-15 07:24:40,463] INFO : Plotted jet ratio +rank0 [2026-11-15 07:24:40,472] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:24:41,906] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:24:42,369] INFO : Plotted number of elements +rank0 [2026-11-15 07:24:43,695] INFO : Plotted jets +rank0 [2026-11-15 07:24:44,689] INFO : Plotted jet ratio +rank0 [2026-11-15 07:24:45,167] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 07:24:45,167] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:28:38,162] INFO : Step 67000/500000 rank0 | Train Loss: 1.1077 | Train Loss Unweighted: 0.6002 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1884 | Regression_eta: 0.0119 | Regression_pt: 0.1817 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2104 | Regression_cos_phi: 0.2010 | Regression_energy: 0.1128 | Regression_eta: 0.1544 | Regression_pt: 0.1087 | Regression_sin_phi: 0.2578 | LR: 2.20e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6809s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0700 | max_loss: 0.0032 | max_backward: 0.1528 | max_optimizer_step: 0.0914 | max_tensorboard_logging: 0.9718 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10182.7 | uss_MB: 9038.5 | child_rss_MB: 23809.8 | allocated_MB: 198.4 | reserved_MB: 21646.0 | inactive_split_MB: 381.6 +rank0 [2026-11-15 07:28:38,165] INFO : Running validation on rank0 +rank0 [2026-11-15 07:28:38,165] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:28:38,165] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:28:38,793] INFO : saved batch0_epoch67000.parquet +rank0 [2026-11-15 07:28:38,914] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:28:39,029] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:28:39,101] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:28:39,170] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:28:39,239] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:28:39,309] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:28:39,380] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:28:39,493] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:28:39,600] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:28:39,703] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:28:39,771] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:28:39,839] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:28:39,909] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:28:39,975] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:28:40,042] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:28:40,148] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:28:40,155] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:28:40,158] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:28:40,160] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:28:40,163] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:28:40,166] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:28:40,168] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:28:40,171] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:28:40,173] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:28:40,176] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:28:40,178] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:28:40,181] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:28:40,183] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:28:40,185] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:28:40,188] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:28:40,190] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:28:47,891] INFO : VALIDATION | Step=67000/500000 | Train Loss=1.1077 | Valid Loss=2.5452 | Train Loss Unweighted=0.6002 | Valid Loss Unweighted=0.8299 | Stale=8 | ETA=2002.5m +rank0 [2026-11-15 07:28:47,892] INFO : split_configs=['1'] +rank0 [2026-11-15 07:28:47,903] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:28:47,909] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:28:48,045] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:28:49,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:28:49,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:28:49,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:28:50,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:28:50,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:28:50,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:28:51,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:28:51,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:28:51,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:28:52,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:28:52,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:28:52,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:28:53,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:28:53,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:28:53,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:28:54,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:28:54,249] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:28:54,253] INFO : split_configs=['1'] +rank0 [2026-11-15 07:28:54,264] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:28:54,267] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:28:54,347] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:28:55,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:28:55,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:28:56,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:28:56,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:28:57,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:28:57,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:28:58,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:28:58,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:28:59,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:28:59,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:28:59,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:29:00,382] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:29:00,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:29:01,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:29:01,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:29:02,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:29:02,261] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:29:02,264] INFO : split_configs=['1'] +rank0 [2026-11-15 07:29:02,275] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:29:02,279] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:29:02,359] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:29:03,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:29:03,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:29:04,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:29:04,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:29:05,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:29:05,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:29:05,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:29:06,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:29:06,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:29:07,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:29:07,555] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:29:07,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:29:08,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:29:08,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:29:09,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:29:09,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:29:09,727] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:29:09,736] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf +rank0 [2026-11-15 07:29:10,949] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf +rank0 [2026-11-15 07:29:11,380] INFO : Plotted number of elements +rank0 [2026-11-15 07:29:12,652] INFO : Plotted jets +rank0 [2026-11-15 07:29:13,679] INFO : Plotted jet ratio +rank0 [2026-11-15 07:29:13,689] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:29:15,846] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:29:16,297] INFO : Plotted number of elements +rank0 [2026-11-15 07:29:17,630] INFO : Plotted jets +rank0 [2026-11-15 07:29:18,661] INFO : Plotted jet ratio +rank0 [2026-11-15 07:29:18,670] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:29:20,084] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:29:20,529] INFO : Plotted number of elements +rank0 [2026-11-15 07:29:21,845] INFO : Plotted jets +rank0 [2026-11-15 07:29:22,843] INFO : Plotted jet ratio +rank0 [2026-11-15 07:31:19,867] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 07:31:19,867] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:33:17,439] INFO : Step 68000/500000 rank0 | Train Loss: 1.7513 | Train Loss Unweighted: 0.7508 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2397 | Regression_cos_phi: 0.0050 | Regression_energy: 0.2416 | Regression_eta: 0.0279 | Regression_pt: 0.2287 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0950 | Classification_binary: 0.2397 | Regression_cos_phi: 0.2918 | Regression_energy: 0.1446 | Regression_eta: 0.3625 | Regression_pt: 0.1368 | Regression_sin_phi: 0.4810 | LR: 2.25e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4313s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0027 | max_forward: 0.0594 | max_loss: 0.0029 | max_backward: 0.1252 | max_optimizer_step: 0.0917 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10196.1 | uss_MB: 9051.9 | child_rss_MB: 23717.6 | allocated_MB: 197.7 | reserved_MB: 18800.0 | inactive_split_MB: 190.3 +rank0 [2026-11-15 07:33:17,441] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-68000.pth +rank0 [2026-11-15 07:33:17,519] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 07:33:17,533] INFO : Running validation on rank0 +rank0 [2026-11-15 07:33:17,534] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:33:17,534] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:33:18,163] INFO : saved batch0_epoch68000.parquet +rank0 [2026-11-15 07:33:18,287] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:33:18,401] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:33:18,471] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:33:18,539] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:33:18,645] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:33:18,714] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:33:18,781] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:33:18,898] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:33:19,002] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:33:19,109] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:33:19,178] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:33:19,245] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:33:19,311] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:33:19,379] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:33:19,447] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:33:19,553] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:33:19,573] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:33:19,586] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:33:19,590] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:33:19,599] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:33:19,612] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:33:19,615] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:33:19,625] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:33:19,633] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:33:19,635] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:33:19,638] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:33:19,640] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:33:19,643] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:33:19,650] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:33:19,653] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:33:19,655] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:33:27,339] INFO : VALIDATION | Step=68000/500000 | Train Loss=1.7513 | Valid Loss=2.5741 | Train Loss Unweighted=0.7508 | Valid Loss Unweighted=0.8457 | Stale=9 | ETA=1998.1m +rank0 [2026-11-15 07:33:27,339] INFO : split_configs=['1'] +rank0 [2026-11-15 07:33:27,350] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:33:27,358] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:33:27,462] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:33:28,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:33:28,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:33:29,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:33:29,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:33:29,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:33:30,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:33:30,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:33:30,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:33:31,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:33:31,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:33:31,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:33:32,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:33:32,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:33:32,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:33:33,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:33:33,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:33:33,627] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:33:33,630] INFO : split_configs=['1'] +rank0 [2026-11-15 07:33:33,642] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:33:33,645] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:33:33,721] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:33:34,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:33:35,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:33:35,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:33:36,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:33:36,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:33:37,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:33:37,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:33:37,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:33:38,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:33:38,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:33:39,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:33:39,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:33:40,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:33:40,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:33:41,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:33:41,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:33:41,632] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:33:41,636] INFO : split_configs=['1'] +rank0 [2026-11-15 07:33:41,647] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:33:41,650] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:33:41,727] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:33:42,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:33:43,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:33:43,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:33:44,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:33:44,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:33:44,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:33:45,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:33:45,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:33:46,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:33:46,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:33:46,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:33:47,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:33:47,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:33:48,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:33:48,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:33:49,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:33:49,178] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:33:49,187] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf +rank0 [2026-11-15 07:33:50,418] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf +rank0 [2026-11-15 07:33:50,852] INFO : Plotted number of elements +rank0 [2026-11-15 07:33:52,175] INFO : Plotted jets +rank0 [2026-11-15 07:33:53,200] INFO : Plotted jet ratio +rank0 [2026-11-15 07:33:53,211] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:33:54,874] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:33:55,862] INFO : Plotted number of elements +rank0 [2026-11-15 07:33:57,215] INFO : Plotted jets +rank0 [2026-11-15 07:33:58,251] INFO : Plotted jet ratio +rank0 [2026-11-15 07:33:58,261] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:33:59,718] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:34:00,158] INFO : Plotted number of elements +rank0 [2026-11-15 07:34:01,482] INFO : Plotted jets +rank0 [2026-11-15 07:34:02,467] INFO : Plotted jet ratio +rank0 [2026-11-15 07:37:56,492] INFO : Step 69000/500000 rank0 | Train Loss: 1.5408 | Train Loss Unweighted: 0.7317 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2385 | Regression_cos_phi: 0.0057 | Regression_energy: 0.2265 | Regression_eta: 0.0165 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2385 | Regression_cos_phi: 0.3342 | Regression_energy: 0.1355 | Regression_eta: 0.2144 | Regression_pt: 0.1423 | Regression_sin_phi: 0.3923 | LR: 2.30e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4574s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0024 | max_forward: 0.0651 | max_loss: 0.0028 | max_backward: 0.1414 | max_optimizer_step: 0.0921 | max_tensorboard_logging: 0.7611 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10232.5 | uss_MB: 9088.4 | child_rss_MB: 23794.0 | allocated_MB: 193.3 | reserved_MB: 18394.0 | inactive_split_MB: 268.7 +rank0 [2026-11-15 07:37:56,495] INFO : Running validation on rank0 +rank0 [2026-11-15 07:37:56,495] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:37:56,495] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:37:57,126] INFO : saved batch0_epoch69000.parquet +rank0 [2026-11-15 07:37:57,251] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:37:57,370] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:37:57,443] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:37:57,513] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:37:57,580] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:37:57,649] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:37:57,716] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:37:57,827] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:37:57,932] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:37:58,036] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:37:58,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:37:58,176] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:37:58,242] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:37:58,308] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:37:58,376] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:37:58,480] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:37:58,488] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:37:58,491] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:37:58,493] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:37:58,496] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:37:58,499] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:37:58,501] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:37:58,504] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:37:58,519] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:37:58,522] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:37:58,530] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:37:58,533] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:37:58,535] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:37:58,538] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:37:58,541] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:37:58,543] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:38:06,268] INFO : VALIDATION | Step=69000/500000 | Train Loss=1.5408 | Valid Loss=2.5331 | Train Loss Unweighted=0.7317 | Valid Loss Unweighted=0.8283 | Stale=10 | ETA=1993.6m +rank0 [2026-11-15 07:38:06,268] INFO : split_configs=['1'] +rank0 [2026-11-15 07:38:06,279] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:38:06,286] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:38:06,397] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:38:07,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:38:07,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:38:08,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:38:08,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:38:08,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:38:09,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:38:09,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:38:09,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:38:10,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:38:10,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:38:10,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:38:11,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:38:11,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:38:11,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:38:12,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:38:12,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:38:12,684] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:38:12,688] INFO : split_configs=['1'] +rank0 [2026-11-15 07:38:12,703] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:38:12,706] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:38:12,788] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:38:14,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:38:14,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:38:14,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:38:15,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:38:15,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:38:16,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:38:16,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:38:17,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:38:17,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:38:18,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:38:18,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:38:19,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:38:19,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:38:19,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:38:20,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:38:20,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:38:20,917] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:38:20,922] INFO : split_configs=['1'] +rank0 [2026-11-15 07:38:20,938] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:38:20,941] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:38:21,025] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:38:22,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:38:22,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:38:23,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:38:23,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:38:23,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:38:24,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:38:24,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:38:25,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:38:25,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:38:25,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:38:26,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:38:26,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:38:27,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:38:27,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:38:28,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:38:28,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:38:28,549] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:38:28,558] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf +rank0 [2026-11-15 07:38:29,730] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf +rank0 [2026-11-15 07:38:30,155] INFO : Plotted number of elements +rank0 [2026-11-15 07:38:31,460] INFO : Plotted jets +rank0 [2026-11-15 07:38:32,487] INFO : Plotted jet ratio +rank0 [2026-11-15 07:38:32,497] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:38:34,102] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:38:35,098] INFO : Plotted number of elements +rank0 [2026-11-15 07:38:36,461] INFO : Plotted jets +rank0 [2026-11-15 07:38:37,516] INFO : Plotted jet ratio +rank0 [2026-11-15 07:38:37,525] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:38:38,981] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:38:39,445] INFO : Plotted number of elements +rank0 [2026-11-15 07:38:40,785] INFO : Plotted jets +rank0 [2026-11-15 07:38:41,814] INFO : Plotted jet ratio +rank0 [2026-11-15 07:38:42,313] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 07:38:42,313] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:42:35,949] INFO : Step 70000/500000 rank0 | Train Loss: 1.4311 | Train Loss Unweighted: 0.7400 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2326 | Regression_cos_phi: 0.0055 | Regression_energy: 0.2413 | Regression_eta: 0.0182 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0819 | Classification_binary: 0.2326 | Regression_cos_phi: 0.3227 | Regression_energy: 0.1444 | Regression_eta: 0.2358 | Regression_pt: 0.1422 | Regression_sin_phi: 0.2714 | LR: 2.35e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4952s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0026 | max_forward: 0.0659 | max_loss: 0.0029 | max_backward: 0.1421 | max_optimizer_step: 0.1024 | max_tensorboard_logging: 0.7785 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10323.3 | uss_MB: 9179.2 | child_rss_MB: 23859.9 | allocated_MB: 194.6 | reserved_MB: 17728.0 | inactive_split_MB: 273.4 +rank0 [2026-11-15 07:42:35,951] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-70000.pth +rank0 [2026-11-15 07:42:36,017] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 07:42:36,030] INFO : Running validation on rank0 +rank0 [2026-11-15 07:42:36,030] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:42:36,030] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:42:36,654] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-15 07:42:36,775] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:42:36,889] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:42:36,960] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:42:37,028] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:42:37,096] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:42:37,163] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:42:37,229] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:42:37,339] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:42:37,443] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:42:37,545] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:42:37,614] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:42:37,681] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:42:37,747] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:42:37,816] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:42:37,884] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:42:37,990] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:42:38,010] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:42:38,023] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:42:38,032] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:42:38,035] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:42:38,038] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:42:38,040] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:42:38,049] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:42:38,051] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:42:38,054] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:42:38,057] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:42:38,059] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:42:38,062] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:42:38,064] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:42:38,067] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:42:38,069] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:42:45,739] INFO : VALIDATION | Step=70000/500000 | Train Loss=1.4311 | Valid Loss=2.5769 | Train Loss Unweighted=0.7400 | Valid Loss Unweighted=0.8331 | Stale=11 | ETA=1989.2m +rank0 [2026-11-15 07:42:45,740] INFO : split_configs=['1'] +rank0 [2026-11-15 07:42:45,750] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:42:45,757] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:42:45,868] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:42:46,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:42:47,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:42:47,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:42:47,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:42:48,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:42:48,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:42:48,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:42:49,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:42:49,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:42:50,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:42:50,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:42:50,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:42:51,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:42:51,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:42:51,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:42:52,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:42:52,129] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:42:52,132] INFO : split_configs=['1'] +rank0 [2026-11-15 07:42:52,143] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:42:52,147] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:42:52,230] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:42:53,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:42:53,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:42:54,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:42:54,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:42:55,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:42:55,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:42:56,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:42:56,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:42:57,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:42:57,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:42:57,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:42:58,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:42:58,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:42:59,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:42:59,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:43:00,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:43:00,271] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:43:00,274] INFO : split_configs=['1'] +rank0 [2026-11-15 07:43:00,285] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:43:00,288] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:43:00,370] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:43:01,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:43:01,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:43:02,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:43:02,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:43:03,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:43:03,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:43:03,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:43:04,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:43:04,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:43:05,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:43:05,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:43:06,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:43:06,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:43:06,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:43:07,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:43:07,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:43:07,855] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:43:07,863] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf +rank0 [2026-11-15 07:43:09,060] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf +rank0 [2026-11-15 07:43:09,482] INFO : Plotted number of elements +rank0 [2026-11-15 07:43:10,778] INFO : Plotted jets +rank0 [2026-11-15 07:43:11,799] INFO : Plotted jet ratio +rank0 [2026-11-15 07:43:11,808] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:43:13,459] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:43:13,899] INFO : Plotted number of elements +rank0 [2026-11-15 07:43:15,800] INFO : Plotted jets +rank0 [2026-11-15 07:43:16,839] INFO : Plotted jet ratio +rank0 [2026-11-15 07:43:16,848] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:43:18,322] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:43:18,761] INFO : Plotted number of elements +rank0 [2026-11-15 07:43:20,101] INFO : Plotted jets +rank0 [2026-11-15 07:43:21,092] INFO : Plotted jet ratio +rank0 [2026-11-15 07:45:17,384] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 07:45:17,384] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:47:14,796] INFO : Step 71000/500000 rank0 | Train Loss: 1.3493 | Train Loss Unweighted: 0.8246 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2303 | Regression_cos_phi: 0.0050 | Regression_energy: 0.2904 | Regression_eta: 0.0100 | Regression_pt: 0.2846 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2303 | Regression_cos_phi: 0.2957 | Regression_energy: 0.1738 | Regression_eta: 0.1295 | Regression_pt: 0.1702 | Regression_sin_phi: 0.2518 | LR: 2.40e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5103s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0681 | max_loss: 0.0028 | max_backward: 0.1440 | max_optimizer_step: 0.0920 | max_tensorboard_logging: 0.7528 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10415.9 | uss_MB: 9271.7 | child_rss_MB: 23714.8 | allocated_MB: 199.1 | reserved_MB: 20906.0 | inactive_split_MB: 120.9 +rank0 [2026-11-15 07:47:14,801] INFO : Running validation on rank0 +rank0 [2026-11-15 07:47:14,801] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:47:14,801] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:47:15,433] INFO : saved batch0_epoch71000.parquet +rank0 [2026-11-15 07:47:15,561] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:47:15,676] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:47:15,747] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:47:15,814] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:47:15,880] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:47:15,947] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:47:16,014] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:47:16,129] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:47:16,235] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:47:16,343] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:47:16,412] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:47:16,481] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:47:16,551] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:47:16,619] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:47:16,687] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:47:16,792] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:47:16,800] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:47:16,803] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:47:16,805] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:47:16,808] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:47:16,810] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:47:16,813] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:47:16,815] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:47:16,818] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:47:16,820] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:47:16,823] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:47:16,825] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:47:16,828] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:47:16,830] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:47:16,833] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:47:16,835] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:47:24,555] INFO : VALIDATION | Step=71000/500000 | Train Loss=1.3493 | Valid Loss=2.5839 | Train Loss Unweighted=0.8246 | Valid Loss Unweighted=0.8257 | Stale=12 | ETA=1984.7m +rank0 [2026-11-15 07:47:24,555] INFO : split_configs=['1'] +rank0 [2026-11-15 07:47:24,566] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:47:24,573] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:47:24,679] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:47:25,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:47:26,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:47:26,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:47:26,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:47:27,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:47:27,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:47:27,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:47:28,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:47:28,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:47:28,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:47:29,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:47:29,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:47:29,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:47:30,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:47:30,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:47:30,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:47:30,850] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:47:30,853] INFO : split_configs=['1'] +rank0 [2026-11-15 07:47:30,864] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:47:30,867] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:47:30,943] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:47:32,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:47:32,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:47:32,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:47:33,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:47:33,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:47:34,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:47:34,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:47:35,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:47:35,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:47:36,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:47:36,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:47:36,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:47:37,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:47:37,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:47:38,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:47:38,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:47:38,860] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:47:38,864] INFO : split_configs=['1'] +rank0 [2026-11-15 07:47:38,898] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:47:38,902] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:47:38,980] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:47:40,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:47:40,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:47:40,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:47:41,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:47:41,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:47:42,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:47:42,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:47:42,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:47:43,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:47:43,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:47:44,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:47:44,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:47:44,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:47:45,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:47:45,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:47:46,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:47:46,305] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 07:47:46,313] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf +rank0 [2026-11-15 07:47:47,512] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf +rank0 [2026-11-15 07:47:47,938] INFO : Plotted number of elements +rank0 [2026-11-15 07:47:49,241] INFO : Plotted jets +rank0 [2026-11-15 07:47:50,259] INFO : Plotted jet ratio +rank0 [2026-11-15 07:47:50,270] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:47:51,927] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:47:52,367] INFO : Plotted number of elements +rank0 [2026-11-15 07:47:54,276] INFO : Plotted jets +rank0 [2026-11-15 07:47:55,307] INFO : Plotted jet ratio +rank0 [2026-11-15 07:47:55,316] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:47:56,760] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:47:57,199] INFO : Plotted number of elements +rank0 [2026-11-15 07:47:58,527] INFO : Plotted jets +rank0 [2026-11-15 07:47:59,515] INFO : Plotted jet ratio +rank0 [2026-11-15 07:51:53,276] INFO : Step 72000/500000 rank0 | Train Loss: 2.3092 | Train Loss Unweighted: 0.7606 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2382 | Regression_cos_phi: 0.0118 | Regression_energy: 0.2363 | Regression_eta: 0.0337 | Regression_pt: 0.2310 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2382 | Regression_cos_phi: 0.6925 | Regression_energy: 0.1414 | Regression_eta: 0.4375 | Regression_pt: 0.1382 | Regression_sin_phi: 0.5802 | LR: 2.45e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4759s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0026 | max_forward: 0.0698 | max_loss: 0.0030 | max_backward: 0.1533 | max_optimizer_step: 0.0928 | max_tensorboard_logging: 0.7604 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 10461.5 | uss_MB: 9317.4 | child_rss_MB: 23745.1 | allocated_MB: 197.4 | reserved_MB: 19668.0 | inactive_split_MB: 438.6 +rank0 [2026-11-15 07:51:53,279] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-72000.pth +rank0 [2026-11-15 07:51:53,345] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 07:51:53,357] INFO : Running validation on rank0 +rank0 [2026-11-15 07:51:53,358] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:51:53,358] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:51:53,984] INFO : saved batch0_epoch72000.parquet +rank0 [2026-11-15 07:51:54,106] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:51:54,220] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:51:54,290] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:51:54,359] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:51:54,427] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:51:54,496] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:51:54,565] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:51:54,680] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:51:54,787] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:51:54,889] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:51:54,957] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:51:55,024] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:51:55,092] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:51:55,160] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:51:55,226] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:51:55,330] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:51:55,337] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:51:55,340] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:51:55,343] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:51:55,345] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:51:55,348] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:51:55,350] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:51:55,353] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:51:55,356] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:51:55,358] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:51:55,360] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:51:55,363] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:51:55,365] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:51:55,368] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:51:55,370] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:51:55,373] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:52:03,071] INFO : VALIDATION | Step=72000/500000 | Train Loss=2.3092 | Valid Loss=2.5534 | Train Loss Unweighted=0.7606 | Valid Loss Unweighted=0.8394 | Stale=13 | ETA=1980.2m +rank0 [2026-11-15 07:52:03,072] INFO : split_configs=['1'] +rank0 [2026-11-15 07:52:03,083] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:52:03,091] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:52:03,229] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:52:04,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:52:04,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:52:04,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:52:05,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:52:05,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:52:06,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:52:06,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:52:06,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:52:07,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:52:07,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:52:07,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:52:08,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:52:08,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:52:08,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:52:09,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:52:09,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:52:09,518] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:52:09,521] INFO : split_configs=['1'] +rank0 [2026-11-15 07:52:09,532] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:52:09,536] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:52:09,617] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:52:10,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:52:11,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:52:11,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:52:12,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:52:12,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:52:13,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:52:13,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:52:13,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:52:14,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:52:14,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:52:15,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:52:15,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:52:16,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:52:16,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:52:17,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:52:17,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:52:17,658] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:52:17,662] INFO : split_configs=['1'] +rank0 [2026-11-15 07:52:17,672] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:52:17,676] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:52:17,759] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:52:18,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:52:19,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:52:19,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:52:20,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:52:20,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:52:21,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:52:21,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:52:21,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:52:22,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:52:22,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:52:23,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:52:23,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:52:23,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:52:24,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:52:24,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:52:25,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:52:25,208] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:52:25,215] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf +rank0 [2026-11-15 07:52:26,402] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf +rank0 [2026-11-15 07:52:26,827] INFO : Plotted number of elements +rank0 [2026-11-15 07:52:28,110] INFO : Plotted jets +rank0 [2026-11-15 07:52:29,129] INFO : Plotted jet ratio +rank0 [2026-11-15 07:52:29,140] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:52:30,833] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:52:31,825] INFO : Plotted number of elements +rank0 [2026-11-15 07:52:33,156] INFO : Plotted jets +rank0 [2026-11-15 07:52:34,187] INFO : Plotted jet ratio +rank0 [2026-11-15 07:52:34,196] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:52:35,650] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:52:36,116] INFO : Plotted number of elements +rank0 [2026-11-15 07:52:37,448] INFO : Plotted jets +rank0 [2026-11-15 07:52:38,462] INFO : Plotted jet ratio +rank0 [2026-11-15 07:52:38,941] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 07:52:38,941] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:56:31,593] INFO : Step 73000/500000 rank0 | Train Loss: 2.1618 | Train Loss Unweighted: 0.7444 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2273 | Regression_cos_phi: 0.0087 | Regression_energy: 0.2377 | Regression_eta: 0.0379 | Regression_pt: 0.2238 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1125 | Classification_binary: 0.2273 | Regression_cos_phi: 0.5097 | Regression_energy: 0.1422 | Regression_eta: 0.4926 | Regression_pt: 0.1339 | Regression_sin_phi: 0.5436 | LR: 2.50e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4289s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0024 | max_forward: 0.0548 | max_loss: 0.0029 | max_backward: 0.1157 | max_optimizer_step: 0.0946 | max_tensorboard_logging: 0.7384 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10537.4 | uss_MB: 9393.3 | child_rss_MB: 23841.6 | allocated_MB: 196.9 | reserved_MB: 17826.0 | inactive_split_MB: 337.1 +rank0 [2026-11-15 07:56:31,596] INFO : Running validation on rank0 +rank0 [2026-11-15 07:56:31,597] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:56:31,597] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:56:32,226] INFO : saved batch0_epoch73000.parquet +rank0 [2026-11-15 07:56:32,347] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:56:32,463] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:56:32,533] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:56:32,600] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:56:32,666] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:56:32,732] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:56:32,800] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:56:32,909] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:56:33,014] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:56:33,120] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:56:33,189] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:56:33,257] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:56:33,325] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:56:33,393] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:56:33,462] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:56:33,567] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:56:33,586] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:56:33,599] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:56:33,602] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:56:33,611] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:56:33,625] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:56:33,628] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:56:33,633] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:56:33,636] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:56:33,638] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:56:33,641] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:56:33,643] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:56:33,645] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:56:33,648] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:56:33,651] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:56:33,654] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:56:41,326] INFO : VALIDATION | Step=73000/500000 | Train Loss=2.1618 | Valid Loss=2.5812 | Train Loss Unweighted=0.7444 | Valid Loss Unweighted=0.8445 | Stale=14 | ETA=1975.6m +rank0 [2026-11-15 07:56:41,328] INFO : split_configs=['1'] +rank0 [2026-11-15 07:56:41,338] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:56:41,346] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:56:41,458] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:56:42,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:56:42,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:56:43,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:56:43,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:56:43,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:56:44,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:56:44,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:56:44,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:56:45,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:56:45,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:56:45,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:56:46,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:56:46,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:56:46,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:56:47,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:56:47,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:56:47,708] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:56:47,712] INFO : split_configs=['1'] +rank0 [2026-11-15 07:56:47,723] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:56:47,726] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:56:47,808] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:56:49,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:56:49,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:56:49,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:56:50,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:56:50,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:56:51,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:56:51,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:56:52,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:56:52,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:56:53,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:56:53,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:56:53,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:56:54,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:56:54,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:56:55,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:56:55,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:56:55,861] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:56:55,864] INFO : split_configs=['1'] +rank0 [2026-11-15 07:56:55,875] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:56:55,879] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:56:55,963] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:56:57,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:56:57,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:56:57,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:56:58,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:56:58,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:56:59,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:56:59,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:56:59,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:57:00,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:57:00,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:57:01,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:57:01,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:57:02,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:57:02,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:57:02,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:57:03,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:57:03,393] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:57:03,401] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf +rank0 [2026-11-15 07:57:04,617] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf +rank0 [2026-11-15 07:57:05,045] INFO : Plotted number of elements +rank0 [2026-11-15 07:57:06,358] INFO : Plotted jets +rank0 [2026-11-15 07:57:07,380] INFO : Plotted jet ratio +rank0 [2026-11-15 07:57:07,391] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:57:09,611] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:57:10,064] INFO : Plotted number of elements +rank0 [2026-11-15 07:57:11,433] INFO : Plotted jets +rank0 [2026-11-15 07:57:12,488] INFO : Plotted jet ratio +rank0 [2026-11-15 07:57:12,497] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:57:14,003] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:57:14,448] INFO : Plotted number of elements +rank0 [2026-11-15 07:57:15,806] INFO : Plotted jets +rank0 [2026-11-15 07:57:16,796] INFO : Plotted jet ratio +rank0 [2026-11-15 07:59:13,999] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 07:59:13,999] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:01:12,102] INFO : Step 74000/500000 rank0 | Train Loss: 1.6976 | Train Loss Unweighted: 0.6749 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2263 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2121 | Regression_eta: 0.0185 | Regression_pt: 0.2026 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.2263 | Regression_cos_phi: 0.5385 | Regression_energy: 0.1270 | Regression_eta: 0.2407 | Regression_pt: 0.1212 | Regression_sin_phi: 0.3677 | LR: 2.55e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5046s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0625 | max_loss: 0.0029 | max_backward: 0.1338 | max_optimizer_step: 0.0963 | max_tensorboard_logging: 0.7545 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10645.0 | uss_MB: 9500.8 | child_rss_MB: 23865.6 | allocated_MB: 195.8 | reserved_MB: 20426.0 | inactive_split_MB: 158.2 +rank0 [2026-11-15 08:01:12,104] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-74000.pth +rank0 [2026-11-15 08:01:12,172] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 08:01:12,185] INFO : Running validation on rank0 +rank0 [2026-11-15 08:01:12,185] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:01:12,185] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:01:12,818] INFO : saved batch0_epoch74000.parquet +rank0 [2026-11-15 08:01:12,948] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:01:13,069] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:01:13,140] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:01:13,208] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:01:13,276] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:01:13,345] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:01:13,414] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:01:13,526] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:01:13,632] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:01:13,735] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:01:13,805] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:01:13,873] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:01:13,941] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:01:14,008] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:01:14,077] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:01:14,184] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:01:14,227] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:01:14,230] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:01:14,232] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:01:14,268] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:01:14,295] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:01:14,298] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:01:14,325] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:01:14,330] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:01:14,332] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:01:14,335] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:01:14,337] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:01:14,340] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:01:14,342] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:01:14,349] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:01:14,351] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:01:22,043] INFO : VALIDATION | Step=74000/500000 | Train Loss=1.6976 | Valid Loss=2.5508 | Train Loss Unweighted=0.6749 | Valid Loss Unweighted=0.8343 | Stale=15 | ETA=1971.3m +rank0 [2026-11-15 08:01:22,044] INFO : split_configs=['1'] +rank0 [2026-11-15 08:01:22,054] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:01:22,062] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:01:22,167] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:01:23,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:01:23,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:01:23,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:01:24,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:01:24,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:01:24,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:01:25,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:01:25,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:01:25,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:01:26,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:01:26,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:01:26,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:01:27,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:01:27,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:01:27,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:01:28,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:01:28,396] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:01:28,399] INFO : split_configs=['1'] +rank0 [2026-11-15 08:01:28,410] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:01:28,413] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:01:28,492] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:01:29,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:01:30,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:01:30,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:01:31,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:01:31,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:01:31,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:01:32,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:01:32,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:01:33,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:01:33,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:01:34,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:01:34,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:01:35,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:01:35,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:01:35,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:01:36,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:01:36,438] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:01:36,441] INFO : split_configs=['1'] +rank0 [2026-11-15 08:01:36,452] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:01:36,455] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:01:36,535] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:01:37,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:01:38,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:01:38,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:01:38,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:01:39,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:01:39,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:01:40,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:01:40,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:01:40,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:01:41,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:01:41,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:01:42,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:01:42,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:01:42,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:01:43,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:01:43,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:01:43,900] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:01:43,909] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf +rank0 [2026-11-15 08:01:45,113] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf +rank0 [2026-11-15 08:01:45,543] INFO : Plotted number of elements +rank0 [2026-11-15 08:01:46,827] INFO : Plotted jets +rank0 [2026-11-15 08:01:47,859] INFO : Plotted jet ratio +rank0 [2026-11-15 08:01:47,869] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:01:49,557] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:01:49,995] INFO : Plotted number of elements +rank0 [2026-11-15 08:01:51,904] INFO : Plotted jets +rank0 [2026-11-15 08:01:52,942] INFO : Plotted jet ratio +rank0 [2026-11-15 08:01:52,951] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:01:54,398] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:01:54,836] INFO : Plotted number of elements +rank0 [2026-11-15 08:01:56,165] INFO : Plotted jets +rank0 [2026-11-15 08:01:57,147] INFO : Plotted jet ratio +rank0 [2026-11-15 08:05:50,676] INFO : Step 75000/500000 rank0 | Train Loss: 1.5745 | Train Loss Unweighted: 0.7623 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2283 | Regression_cos_phi: 0.0084 | Regression_energy: 0.2569 | Regression_eta: 0.0147 | Regression_pt: 0.2491 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2283 | Regression_cos_phi: 0.4912 | Regression_energy: 0.1537 | Regression_eta: 0.1912 | Regression_pt: 0.1490 | Regression_sin_phi: 0.2912 | LR: 2.60e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4887s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0027 | max_forward: 0.0637 | max_loss: 0.0034 | max_backward: 0.1351 | max_optimizer_step: 0.0954 | max_tensorboard_logging: 0.7848 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 10607.3 | uss_MB: 9463.2 | child_rss_MB: 23811.9 | allocated_MB: 195.9 | reserved_MB: 17986.0 | inactive_split_MB: 240.1 +rank0 [2026-11-15 08:05:50,678] INFO : Running validation on rank0 +rank0 [2026-11-15 08:05:50,678] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:05:50,678] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:05:51,313] INFO : saved batch0_epoch75000.parquet +rank0 [2026-11-15 08:05:51,437] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:05:51,552] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:05:51,623] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:05:51,691] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:05:51,759] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:05:51,828] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:05:51,897] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:05:52,009] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:05:52,119] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:05:52,225] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:05:52,295] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:05:52,363] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:05:52,429] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:05:52,497] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:05:52,564] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:05:52,669] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:05:52,676] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:05:52,679] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:05:52,687] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:05:52,690] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:05:52,693] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:05:52,695] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:05:52,697] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:05:52,700] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:05:52,702] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:05:52,705] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:05:52,707] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:05:52,710] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:05:52,712] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:05:52,715] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:05:52,717] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:06:00,442] INFO : VALIDATION | Step=75000/500000 | Train Loss=1.5745 | Valid Loss=2.5886 | Train Loss Unweighted=0.7623 | Valid Loss Unweighted=0.8278 | Stale=16 | ETA=1966.7m +rank0 [2026-11-15 08:06:00,442] INFO : split_configs=['1'] +rank0 [2026-11-15 08:06:00,454] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:06:00,460] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:06:00,578] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:06:01,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:06:02,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:06:02,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:06:02,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:06:03,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:06:03,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:06:03,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:06:04,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:06:04,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:06:04,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:06:05,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:06:05,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:06:05,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:06:06,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:06:06,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:06:06,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:06:06,940] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:06:06,943] INFO : split_configs=['1'] +rank0 [2026-11-15 08:06:06,954] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:06:06,958] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:06:07,042] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:06:08,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:06:08,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:06:09,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:06:09,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:06:10,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:06:10,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:06:10,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:06:11,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:06:11,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:06:12,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:06:12,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:06:13,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:06:13,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:06:14,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:06:14,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:06:15,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:06:15,145] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:06:15,148] INFO : split_configs=['1'] +rank0 [2026-11-15 08:06:15,158] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:06:15,162] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:06:15,248] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:06:16,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:06:16,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:06:17,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:06:17,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:06:18,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:06:18,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:06:18,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:06:19,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:06:19,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:06:20,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:06:20,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:06:20,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:06:21,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:06:21,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:06:22,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:06:22,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:06:22,678] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:06:22,686] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf +rank0 [2026-11-15 08:06:23,896] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf +rank0 [2026-11-15 08:06:24,333] INFO : Plotted number of elements +rank0 [2026-11-15 08:06:25,606] INFO : Plotted jets +rank0 [2026-11-15 08:06:26,645] INFO : Plotted jet ratio +rank0 [2026-11-15 08:06:26,655] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:06:28,280] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:06:29,295] INFO : Plotted number of elements +rank0 [2026-11-15 08:06:30,627] INFO : Plotted jets +rank0 [2026-11-15 08:06:31,670] INFO : Plotted jet ratio +rank0 [2026-11-15 08:06:31,679] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:06:33,160] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:06:33,597] INFO : Plotted number of elements +rank0 [2026-11-15 08:06:34,926] INFO : Plotted jets +rank0 [2026-11-15 08:06:35,910] INFO : Plotted jet ratio +rank0 [2026-11-15 08:06:36,406] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 08:06:36,406] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:10:31,141] INFO : Step 76000/500000 rank0 | Train Loss: 2.1588 | Train Loss Unweighted: 0.7861 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2232 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2658 | Regression_eta: 0.0345 | Regression_pt: 0.2439 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2232 | Regression_cos_phi: 0.5431 | Regression_energy: 0.1591 | Regression_eta: 0.4482 | Regression_pt: 0.1459 | Regression_sin_phi: 0.5787 | LR: 2.65e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.5725s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0546 | max_loss: 0.0029 | max_backward: 0.1149 | max_optimizer_step: 0.0979 | max_tensorboard_logging: 1.8890 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10673.2 | uss_MB: 9529.0 | child_rss_MB: 23733.4 | allocated_MB: 196.4 | reserved_MB: 18368.0 | inactive_split_MB: 257.6 +rank0 [2026-11-15 08:10:31,144] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-76000.pth +rank0 [2026-11-15 08:10:31,214] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 08:10:31,229] INFO : Running validation on rank0 +rank0 [2026-11-15 08:10:31,229] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:10:31,230] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:10:31,855] INFO : saved batch0_epoch76000.parquet +rank0 [2026-11-15 08:10:31,975] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:10:32,092] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:10:32,161] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:10:32,229] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:10:32,295] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:10:32,362] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:10:32,429] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:10:32,537] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:10:32,641] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:10:32,744] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:10:32,813] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:10:32,881] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:10:32,949] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:10:33,019] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:10:33,087] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:10:33,193] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:10:33,213] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:10:33,225] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:10:33,228] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:10:33,231] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:10:33,238] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:10:33,240] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:10:33,243] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:10:33,245] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:10:33,248] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:10:33,250] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:10:33,253] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:10:33,255] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:10:33,258] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:10:33,260] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:10:33,263] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:10:40,946] INFO : VALIDATION | Step=76000/500000 | Train Loss=2.1588 | Valid Loss=2.6047 | Train Loss Unweighted=0.7861 | Valid Loss Unweighted=0.8316 | Stale=17 | ETA=1962.4m +rank0 [2026-11-15 08:10:40,947] INFO : split_configs=['1'] +rank0 [2026-11-15 08:10:40,963] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:10:40,972] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:10:41,089] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:10:42,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:10:42,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:10:42,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:10:43,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:10:43,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:10:43,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:10:44,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:10:44,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:10:44,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:10:45,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:10:45,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:10:45,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:10:46,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:10:46,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:10:46,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:10:47,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:10:47,361] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:10:47,364] INFO : split_configs=['1'] +rank0 [2026-11-15 08:10:47,382] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:10:47,385] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:10:47,469] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:10:48,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:10:49,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:10:49,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:10:50,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:10:50,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:10:50,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:10:51,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:10:51,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:10:52,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:10:52,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:10:53,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:10:53,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:10:54,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:10:54,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:10:54,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:10:55,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:10:55,550] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:10:55,553] INFO : split_configs=['1'] +rank0 [2026-11-15 08:10:55,571] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:10:55,576] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:10:55,662] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:10:56,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:10:57,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:10:57,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:10:58,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:10:58,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:10:58,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:10:59,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:10:59,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:11:00,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:11:00,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:11:00,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:11:01,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:11:01,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:11:02,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:11:02,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:11:03,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:11:03,166] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:11:03,174] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf +rank0 [2026-11-15 08:11:04,364] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf +rank0 [2026-11-15 08:11:04,793] INFO : Plotted number of elements +rank0 [2026-11-15 08:11:06,110] INFO : Plotted jets +rank0 [2026-11-15 08:11:07,137] INFO : Plotted jet ratio +rank0 [2026-11-15 08:11:07,162] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:11:08,978] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:11:10,002] INFO : Plotted number of elements +rank0 [2026-11-15 08:11:11,351] INFO : Plotted jets +rank0 [2026-11-15 08:11:12,419] INFO : Plotted jet ratio +rank0 [2026-11-15 08:11:12,428] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:11:13,918] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:11:14,367] INFO : Plotted number of elements +rank0 [2026-11-15 08:11:15,726] INFO : Plotted jets +rank0 [2026-11-15 08:11:16,722] INFO : Plotted jet ratio +rank0 [2026-11-15 08:13:13,338] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 08:13:13,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:15:11,607] INFO : Step 77000/500000 rank0 | Train Loss: 1.7834 | Train Loss Unweighted: 0.7779 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2328 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2549 | Regression_eta: 0.0127 | Regression_pt: 0.2611 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1095 | Classification_binary: 0.2328 | Regression_cos_phi: 0.5244 | Regression_energy: 0.1525 | Regression_eta: 0.1651 | Regression_pt: 0.1562 | Regression_sin_phi: 0.4428 | LR: 2.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4213s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0582 | max_loss: 0.0029 | max_backward: 0.1203 | max_optimizer_step: 0.0949 | max_tensorboard_logging: 1.7348 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9322.9 | uss_MB: 8178.8 | child_rss_MB: 23821.5 | allocated_MB: 195.2 | reserved_MB: 18500.0 | inactive_split_MB: 292.8 +rank0 [2026-11-15 08:15:11,612] INFO : Running validation on rank0 +rank0 [2026-11-15 08:15:11,612] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:15:11,612] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:15:12,245] INFO : saved batch0_epoch77000.parquet +rank0 [2026-11-15 08:15:12,375] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:15:12,494] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:15:12,566] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:15:12,639] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:15:12,709] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:15:12,777] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:15:12,846] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:15:12,957] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:15:13,065] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:15:13,170] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:15:13,240] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:15:13,308] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:15:13,375] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:15:13,443] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:15:13,510] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:15:13,614] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:15:13,621] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:15:13,624] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:15:13,627] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:15:13,629] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:15:13,632] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:15:13,634] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:15:13,638] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:15:13,640] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:15:13,643] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:15:13,645] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:15:13,648] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:15:13,650] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:15:13,652] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:15:13,655] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:15:13,657] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:15:21,399] INFO : VALIDATION | Step=77000/500000 | Train Loss=1.7834 | Valid Loss=2.5878 | Train Loss Unweighted=0.7779 | Valid Loss Unweighted=0.8254 | Stale=18 | ETA=1958.0m +rank0 [2026-11-15 08:15:21,399] INFO : split_configs=['1'] +rank0 [2026-11-15 08:15:21,417] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:15:21,426] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:15:21,531] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:15:22,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:15:22,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:15:23,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:15:23,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:15:23,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:15:24,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:15:24,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:15:24,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:15:25,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:15:25,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:15:26,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:15:26,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:15:26,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:15:27,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:15:27,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:15:27,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:15:27,852] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:15:27,855] INFO : split_configs=['1'] +rank0 [2026-11-15 08:15:27,875] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:15:27,878] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:15:27,956] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:15:29,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:15:29,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:15:30,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:15:30,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:15:30,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:15:31,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:15:31,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:15:32,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:15:32,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:15:33,171] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:15:33,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:15:34,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:15:34,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:15:34,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:15:35,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:15:35,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:15:35,954] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:15:35,958] INFO : split_configs=['1'] +rank0 [2026-11-15 08:15:35,975] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:15:35,979] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:15:36,062] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:15:37,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:15:37,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:15:38,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:15:38,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:15:38,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:15:39,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:15:39,701] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:15:40,107] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:15:40,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:15:40,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:15:41,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:15:41,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:15:42,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:15:42,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:15:43,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:15:43,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:15:43,533] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:15:43,553] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf +rank0 [2026-11-15 08:15:44,784] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf +rank0 [2026-11-15 08:15:45,220] INFO : Plotted number of elements +rank0 [2026-11-15 08:15:46,536] INFO : Plotted jets +rank0 [2026-11-15 08:15:47,560] INFO : Plotted jet ratio +rank0 [2026-11-15 08:15:47,585] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:15:49,330] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:15:50,342] INFO : Plotted number of elements +rank0 [2026-11-15 08:15:51,708] INFO : Plotted jets +rank0 [2026-11-15 08:15:52,753] INFO : Plotted jet ratio +rank0 [2026-11-15 08:15:52,763] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:15:54,281] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:15:54,723] INFO : Plotted number of elements +rank0 [2026-11-15 08:15:56,064] INFO : Plotted jets +rank0 [2026-11-15 08:15:57,057] INFO : Plotted jet ratio +rank0 [2026-11-15 08:19:52,839] INFO : Step 78000/500000 rank0 | Train Loss: 1.6543 | Train Loss Unweighted: 0.8745 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2238 | Regression_cos_phi: 0.0057 | Regression_energy: 0.3213 | Regression_eta: 0.0268 | Regression_pt: 0.2919 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0877 | Classification_binary: 0.2238 | Regression_cos_phi: 0.3377 | Regression_energy: 0.1923 | Regression_eta: 0.3476 | Regression_pt: 0.1746 | Regression_sin_phi: 0.2906 | LR: 2.75e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.8878s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0585 | max_loss: 0.0029 | max_backward: 0.1211 | max_optimizer_step: 0.0955 | max_tensorboard_logging: 2.1938 | max_tensorboard_flush: 0.0008 | Memory: rss_MB: 10814.0 | uss_MB: 9669.8 | child_rss_MB: 23777.0 | allocated_MB: 196.9 | reserved_MB: 20432.0 | inactive_split_MB: 287.1 +rank0 [2026-11-15 08:19:52,846] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-78000.pth +rank0 [2026-11-15 08:19:52,927] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 08:19:52,942] INFO : Running validation on rank0 +rank0 [2026-11-15 08:19:52,942] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:19:52,942] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:19:53,572] INFO : saved batch0_epoch78000.parquet +rank0 [2026-11-15 08:19:53,693] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:19:53,807] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:19:53,877] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:19:53,945] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:19:54,012] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:19:54,081] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:19:54,148] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:19:54,263] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:19:54,367] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:19:54,469] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:19:54,538] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:19:54,607] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:19:54,674] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:19:54,743] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:19:54,812] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:19:54,919] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:19:54,938] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:19:54,945] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:19:54,947] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:19:54,951] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:19:54,954] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:19:54,957] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:19:54,959] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:19:54,962] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:19:54,964] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:19:54,966] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:19:54,969] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:19:54,972] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:19:54,974] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:19:54,977] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:19:54,979] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:20:02,691] INFO : VALIDATION | Step=78000/500000 | Train Loss=1.6543 | Valid Loss=2.6549 | Train Loss Unweighted=0.8745 | Valid Loss Unweighted=0.8375 | Stale=19 | ETA=1953.7m +rank0 [2026-11-15 08:20:02,691] INFO : split_configs=['1'] +rank0 [2026-11-15 08:20:02,708] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:20:02,716] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:20:03,429] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:20:04,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:20:04,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:20:05,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:20:05,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:20:05,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:20:06,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:20:06,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:20:06,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:20:07,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:20:07,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:20:08,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:20:08,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:20:08,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:20:09,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:20:09,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:20:09,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:20:09,800] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:20:09,803] INFO : split_configs=['1'] +rank0 [2026-11-15 08:20:09,821] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:20:09,835] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:20:09,921] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:20:11,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:20:11,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:20:12,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:20:12,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:20:13,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:20:13,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:20:13,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:20:14,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:20:14,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:20:15,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:20:15,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:20:16,236] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:20:16,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:20:17,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:20:17,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:20:18,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:20:18,149] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:20:18,153] INFO : split_configs=['1'] +rank0 [2026-11-15 08:20:18,171] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:20:18,185] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:20:18,273] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:20:19,554] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:20:19,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:20:20,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:20:20,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:20:21,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:20:21,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:20:22,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:20:22,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:20:22,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:20:23,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:20:23,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:20:24,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:20:24,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:20:24,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:20:25,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:20:25,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:20:25,838] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:20:26,170] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf +rank0 [2026-11-15 08:20:27,384] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf +rank0 [2026-11-15 08:20:27,839] INFO : Plotted number of elements +rank0 [2026-11-15 08:20:29,111] INFO : Plotted jets +rank0 [2026-11-15 08:20:30,159] INFO : Plotted jet ratio +rank0 [2026-11-15 08:20:30,170] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:20:31,843] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:20:32,289] INFO : Plotted number of elements +rank0 [2026-11-15 08:20:34,201] INFO : Plotted jets +rank0 [2026-11-15 08:20:35,246] INFO : Plotted jet ratio +rank0 [2026-11-15 08:20:35,257] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:20:36,718] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:20:37,156] INFO : Plotted number of elements +rank0 [2026-11-15 08:20:38,488] INFO : Plotted jets +rank0 [2026-11-15 08:20:39,474] INFO : Plotted jet ratio +rank0 [2026-11-15 08:20:39,955] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 08:20:39,955] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:24:34,778] INFO : Step 79000/500000 rank0 | Train Loss: 1.7156 | Train Loss Unweighted: 0.9099 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2634 | Regression_cos_phi: 0.0072 | Regression_energy: 0.3109 | Regression_eta: 0.0124 | Regression_pt: 0.3093 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0955 | Classification_binary: 0.2634 | Regression_cos_phi: 0.4241 | Regression_energy: 0.1861 | Regression_eta: 0.1608 | Regression_pt: 0.1850 | Regression_sin_phi: 0.4008 | LR: 2.80e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.7678s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0590 | max_loss: 0.0028 | max_backward: 0.1241 | max_optimizer_step: 0.0950 | max_tensorboard_logging: 2.0671 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 10902.3 | uss_MB: 9758.2 | child_rss_MB: 23650.8 | allocated_MB: 195.8 | reserved_MB: 18684.0 | inactive_split_MB: 152.2 +rank0 [2026-11-15 08:24:34,780] INFO : Running validation on rank0 +rank0 [2026-11-15 08:24:34,781] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:24:34,781] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:24:35,419] INFO : saved batch0_epoch79000.parquet +rank0 [2026-11-15 08:24:35,541] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:24:35,658] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:24:35,729] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:24:35,797] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:24:35,864] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:24:35,932] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:24:36,000] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:24:36,114] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:24:36,219] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:24:36,321] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:24:36,389] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:24:36,457] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:24:36,522] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:24:36,588] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:24:36,654] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:24:36,759] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:24:36,802] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:24:36,829] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:24:36,832] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:24:36,859] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:24:36,886] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:24:36,889] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:24:36,911] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:24:36,914] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:24:36,916] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:24:36,918] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:24:36,921] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:24:36,926] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:24:36,928] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:24:36,937] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:24:36,941] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:24:44,654] INFO : VALIDATION | Step=79000/500000 | Train Loss=1.7156 | Valid Loss=2.6497 | Train Loss Unweighted=0.9099 | Valid Loss Unweighted=0.8442 | Stale=20 | ETA=1949.4m +rank0 [2026-11-15 08:24:44,656] INFO : split_configs=['1'] +rank0 [2026-11-15 08:24:44,673] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:24:44,682] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:24:44,798] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:24:45,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:24:46,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:24:46,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:24:46,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:24:47,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:24:47,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:24:47,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:24:48,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:24:48,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:24:49,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:24:49,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:24:49,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:24:50,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:24:50,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:24:50,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:24:51,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:24:51,136] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:24:51,140] INFO : split_configs=['1'] +rank0 [2026-11-15 08:24:51,158] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:24:51,165] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:24:51,251] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:24:52,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:24:52,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:24:53,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:24:53,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:24:54,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:24:54,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:24:55,142] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:24:55,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:24:56,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:24:56,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:24:56,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:24:57,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:24:57,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:24:58,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:24:58,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:24:59,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:24:59,309] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:24:59,313] INFO : split_configs=['1'] +rank0 [2026-11-15 08:24:59,329] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:24:59,345] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:24:59,432] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:25:00,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:25:01,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:25:01,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:25:01,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:25:02,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:25:02,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:25:03,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:25:03,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:25:03,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:25:04,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:25:04,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:25:05,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:25:05,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:25:06,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:25:06,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:25:06,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:25:07,011] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:25:07,030] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf +rank0 [2026-11-15 08:25:08,257] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf +rank0 [2026-11-15 08:25:08,684] INFO : Plotted number of elements +rank0 [2026-11-15 08:25:10,012] INFO : Plotted jets +rank0 [2026-11-15 08:25:11,034] INFO : Plotted jet ratio +rank0 [2026-11-15 08:25:11,046] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:25:13,306] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:25:13,757] INFO : Plotted number of elements +rank0 [2026-11-15 08:25:15,125] INFO : Plotted jets +rank0 [2026-11-15 08:25:16,163] INFO : Plotted jet ratio +rank0 [2026-11-15 08:25:16,200] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:25:17,809] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:25:18,257] INFO : Plotted number of elements +rank0 [2026-11-15 08:25:19,606] INFO : Plotted jets +rank0 [2026-11-15 08:25:20,598] INFO : Plotted jet ratio +rank0 [2026-11-15 08:27:16,969] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 08:27:16,969] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:29:15,201] INFO : Step 80000/500000 rank0 | Train Loss: 1.5259 | Train Loss Unweighted: 0.8379 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2344 | Regression_cos_phi: 0.0047 | Regression_energy: 0.2806 | Regression_eta: 0.0311 | Regression_pt: 0.2838 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0848 | Classification_binary: 0.2344 | Regression_cos_phi: 0.2748 | Regression_energy: 0.1680 | Regression_eta: 0.4036 | Regression_pt: 0.1697 | Regression_sin_phi: 0.1906 | LR: 2.85e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.3365s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0027 | max_forward: 0.0609 | max_loss: 0.0029 | max_backward: 0.1302 | max_optimizer_step: 0.0945 | max_tensorboard_logging: 1.6353 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10668.0 | uss_MB: 9530.4 | child_rss_MB: 23834.4 | allocated_MB: 195.5 | reserved_MB: 20714.0 | inactive_split_MB: 236.5 +rank0 [2026-11-15 08:29:15,204] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-80000.pth +rank0 [2026-11-15 08:29:15,275] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 08:29:15,293] INFO : Running validation on rank0 +rank0 [2026-11-15 08:29:15,293] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:29:15,293] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:29:15,941] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-15 08:29:16,064] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:29:16,180] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:29:16,251] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:29:16,319] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:29:16,386] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:29:16,454] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:29:16,522] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:29:16,633] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:29:16,738] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:29:16,841] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:29:16,910] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:29:16,979] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:29:17,047] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:29:17,116] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:29:17,186] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:29:17,294] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:29:17,315] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:29:17,327] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:29:17,330] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:29:17,340] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:29:17,353] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:29:17,356] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:29:17,366] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:29:17,379] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:29:17,382] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:29:17,384] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:29:17,387] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:29:17,389] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:29:17,391] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:29:17,394] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:29:17,397] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:29:25,074] INFO : VALIDATION | Step=80000/500000 | Train Loss=1.5259 | Valid Loss=2.5604 | Train Loss Unweighted=0.8379 | Valid Loss Unweighted=0.8351 | Stale=21 | ETA=1945.0m +rank0 [2026-11-15 08:29:25,074] INFO : split_configs=['1'] +rank0 [2026-11-15 08:29:25,088] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:29:25,097] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:29:25,211] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:29:26,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:29:26,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:29:26,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:29:27,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:29:27,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:29:28,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:29:28,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:29:28,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:29:29,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:29:29,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:29:29,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:29:30,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:29:30,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:29:30,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:29:31,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:29:31,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:29:31,547] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:29:31,551] INFO : split_configs=['1'] +rank0 [2026-11-15 08:29:31,568] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:29:31,581] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:29:31,667] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:29:32,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:29:33,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:29:33,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:29:34,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:29:34,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:29:35,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:29:35,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:29:35,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:29:36,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:29:36,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:29:37,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:29:37,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:29:38,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:29:38,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:29:39,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:29:39,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:29:39,707] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:29:39,710] INFO : split_configs=['1'] +rank0 [2026-11-15 08:29:39,727] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:29:39,731] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:29:39,819] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:29:41,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:29:41,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:29:41,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:29:42,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:29:42,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:29:43,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:29:43,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:29:43,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:29:44,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:29:44,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:29:45,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:29:45,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:29:45,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:29:46,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:29:46,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:29:47,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:29:47,245] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:29:47,253] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf +rank0 [2026-11-15 08:29:48,501] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf +rank0 [2026-11-15 08:29:48,918] INFO : Plotted number of elements +rank0 [2026-11-15 08:29:50,209] INFO : Plotted jets +rank0 [2026-11-15 08:29:51,224] INFO : Plotted jet ratio +rank0 [2026-11-15 08:29:51,235] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:29:52,886] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:29:53,323] INFO : Plotted number of elements +rank0 [2026-11-15 08:29:55,231] INFO : Plotted jets +rank0 [2026-11-15 08:29:56,275] INFO : Plotted jet ratio +rank0 [2026-11-15 08:29:56,284] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:29:57,724] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:29:58,161] INFO : Plotted number of elements +rank0 [2026-11-15 08:29:59,533] INFO : Plotted jets +rank0 [2026-11-15 08:30:00,526] INFO : Plotted jet ratio +rank0 [2026-11-15 08:33:55,712] INFO : Step 81000/500000 rank0 | Train Loss: 1.7250 | Train Loss Unweighted: 0.8072 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2352 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2679 | Regression_eta: 0.0108 | Regression_pt: 0.2775 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0800 | Classification_binary: 0.2352 | Regression_cos_phi: 0.5009 | Regression_energy: 0.1603 | Regression_eta: 0.1401 | Regression_pt: 0.1660 | Regression_sin_phi: 0.4425 | LR: 2.90e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.0514s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0712 | max_loss: 0.0040 | max_backward: 0.1531 | max_optimizer_step: 0.0967 | max_tensorboard_logging: 2.3037 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11126.5 | uss_MB: 9988.9 | child_rss_MB: 23790.1 | allocated_MB: 200.2 | reserved_MB: 21586.0 | inactive_split_MB: 461.8 +rank0 [2026-11-15 08:33:55,714] INFO : Running validation on rank0 +rank0 [2026-11-15 08:33:55,714] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:33:55,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:33:56,361] INFO : saved batch0_epoch81000.parquet +rank0 [2026-11-15 08:33:56,484] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:33:56,601] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:33:56,673] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:33:56,741] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:33:56,810] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:33:56,880] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:33:56,951] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:33:57,061] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:33:57,165] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:33:57,269] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:33:57,338] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:33:57,405] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:33:57,471] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:33:57,537] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:33:57,604] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:33:57,707] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:33:57,714] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:33:57,717] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:33:57,719] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:33:57,722] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:33:57,725] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:33:57,727] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:33:57,730] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:33:57,732] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:33:57,735] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:33:57,737] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:33:57,740] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:33:57,742] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:33:57,744] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:33:57,747] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:33:57,749] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:34:05,462] INFO : VALIDATION | Step=81000/500000 | Train Loss=1.7250 | Valid Loss=2.5705 | Train Loss Unweighted=0.8072 | Valid Loss Unweighted=0.8231 | Stale=22 | ETA=1940.6m +rank0 [2026-11-15 08:34:05,463] INFO : split_configs=['1'] +rank0 [2026-11-15 08:34:05,481] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:34:05,491] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:34:05,610] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:34:06,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:34:07,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:34:07,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:34:07,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:34:08,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:34:08,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:34:08,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:34:09,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:34:09,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:34:09,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:34:10,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:34:10,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:34:10,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:34:11,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:34:11,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:34:11,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:34:11,939] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:34:11,942] INFO : split_configs=['1'] +rank0 [2026-11-15 08:34:11,960] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:34:11,964] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:34:12,051] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:34:13,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:34:13,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:34:14,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:34:14,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:34:15,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:34:15,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:34:15,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:34:16,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:34:16,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:34:17,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:34:17,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:34:18,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:34:18,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:34:19,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:34:19,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:34:20,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:34:20,240] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:34:20,244] INFO : split_configs=['1'] +rank0 [2026-11-15 08:34:20,262] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:34:20,265] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:34:20,354] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:34:21,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:34:22,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:34:22,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:34:22,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:34:23,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:34:23,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:34:24,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:34:24,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:34:24,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:34:25,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:34:25,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:34:26,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:34:26,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:34:26,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:34:27,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:34:27,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:34:27,870] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:34:27,880] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf +rank0 [2026-11-15 08:34:29,112] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf +rank0 [2026-11-15 08:34:29,531] INFO : Plotted number of elements +rank0 [2026-11-15 08:34:30,836] INFO : Plotted jets +rank0 [2026-11-15 08:34:31,854] INFO : Plotted jet ratio +rank0 [2026-11-15 08:34:31,878] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:34:33,515] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:34:33,956] INFO : Plotted number of elements +rank0 [2026-11-15 08:34:35,878] INFO : Plotted jets +rank0 [2026-11-15 08:34:36,916] INFO : Plotted jet ratio +rank0 [2026-11-15 08:34:36,937] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:34:38,398] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:34:38,859] INFO : Plotted number of elements +rank0 [2026-11-15 08:34:40,188] INFO : Plotted jets +rank0 [2026-11-15 08:34:41,186] INFO : Plotted jet ratio +rank0 [2026-11-15 08:34:41,675] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 08:34:41,675] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:38:35,769] INFO : Step 82000/500000 rank0 | Train Loss: 2.7318 | Train Loss Unweighted: 0.8209 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2615 | Regression_cos_phi: 0.0109 | Regression_energy: 0.2478 | Regression_eta: 0.0494 | Regression_pt: 0.2382 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0937 | Classification_binary: 0.2615 | Regression_cos_phi: 0.6382 | Regression_energy: 0.1483 | Regression_eta: 0.6421 | Regression_pt: 0.1425 | Regression_sin_phi: 0.8055 | LR: 2.95e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.7294s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0650 | max_loss: 0.0029 | max_backward: 0.1376 | max_optimizer_step: 0.0970 | max_tensorboard_logging: 1.9982 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11233.9 | uss_MB: 10096.3 | child_rss_MB: 23748.8 | allocated_MB: 198.9 | reserved_MB: 19538.0 | inactive_split_MB: 405.1 +rank0 [2026-11-15 08:38:35,771] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-82000.pth +rank0 [2026-11-15 08:38:35,842] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 08:38:35,857] INFO : Running validation on rank0 +rank0 [2026-11-15 08:38:35,857] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:38:35,857] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:38:36,481] INFO : saved batch0_epoch82000.parquet +rank0 [2026-11-15 08:38:36,603] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:38:36,724] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:38:36,795] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:38:36,865] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:38:36,933] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:38:37,003] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:38:37,073] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:38:37,187] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:38:37,291] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:38:37,393] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:38:37,463] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:38:37,530] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:38:37,595] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:38:37,661] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:38:37,727] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:38:37,830] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:38:37,837] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:38:37,840] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:38:37,843] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:38:37,845] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:38:37,848] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:38:37,851] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:38:37,853] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:38:37,856] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:38:37,858] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:38:37,860] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:38:37,863] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:38:37,865] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:38:37,907] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:38:37,911] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:38:37,913] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:38:45,586] INFO : VALIDATION | Step=82000/500000 | Train Loss=2.7318 | Valid Loss=2.5645 | Train Loss Unweighted=0.8209 | Valid Loss Unweighted=0.8237 | Stale=23 | ETA=1936.2m +rank0 [2026-11-15 08:38:45,587] INFO : split_configs=['1'] +rank0 [2026-11-15 08:38:45,604] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:38:45,616] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:38:45,723] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:38:46,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:38:47,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:38:47,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:38:47,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:38:48,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:38:48,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:38:48,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:38:49,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:38:49,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:38:49,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:38:50,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:38:50,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:38:50,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:38:51,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:38:51,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:38:51,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:38:51,940] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:38:51,943] INFO : split_configs=['1'] +rank0 [2026-11-15 08:38:51,962] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:38:51,965] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:38:52,051] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:38:53,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:38:53,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:38:54,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:38:54,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:38:54,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:38:55,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:38:55,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:38:56,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:38:56,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:38:57,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:38:57,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:38:58,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:38:58,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:38:59,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:38:59,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:38:59,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:39:00,003] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:39:00,007] INFO : split_configs=['1'] +rank0 [2026-11-15 08:39:00,024] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:39:00,027] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:39:00,117] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:39:01,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:39:01,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:39:02,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:39:02,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:39:02,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:39:03,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:39:03,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:39:04,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:39:04,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:39:04,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:39:05,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:39:05,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:39:06,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:39:06,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:39:07,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:39:07,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:39:07,550] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:39:07,558] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf +rank0 [2026-11-15 08:39:08,812] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf +rank0 [2026-11-15 08:39:09,251] INFO : Plotted number of elements +rank0 [2026-11-15 08:39:10,551] INFO : Plotted jets +rank0 [2026-11-15 08:39:11,573] INFO : Plotted jet ratio +rank0 [2026-11-15 08:39:11,586] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:39:13,301] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:39:13,758] INFO : Plotted number of elements +rank0 [2026-11-15 08:39:15,692] INFO : Plotted jets +rank0 [2026-11-15 08:39:16,732] INFO : Plotted jet ratio +rank0 [2026-11-15 08:39:16,743] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:39:18,224] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:39:18,665] INFO : Plotted number of elements +rank0 [2026-11-15 08:39:20,002] INFO : Plotted jets +rank0 [2026-11-15 08:39:20,987] INFO : Plotted jet ratio +rank0 [2026-11-15 08:41:17,503] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 08:41:17,504] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:43:16,419] INFO : Step 83000/500000 rank0 | Train Loss: 1.3647 | Train Loss Unweighted: 0.6794 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2196 | Regression_cos_phi: 0.0035 | Regression_energy: 0.2203 | Regression_eta: 0.0255 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2196 | Regression_cos_phi: 0.2083 | Regression_energy: 0.1319 | Regression_eta: 0.3306 | Regression_pt: 0.1231 | Regression_sin_phi: 0.2761 | LR: 3.00e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5498s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0724 | max_loss: 0.0028 | max_backward: 0.1593 | max_optimizer_step: 0.0975 | max_tensorboard_logging: 1.7920 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 11269.2 | uss_MB: 10131.6 | child_rss_MB: 23846.5 | allocated_MB: 196.9 | reserved_MB: 19362.0 | inactive_split_MB: 303.1 +rank0 [2026-11-15 08:43:16,423] INFO : Running validation on rank0 +rank0 [2026-11-15 08:43:16,424] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:43:16,424] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:43:17,047] INFO : saved batch0_epoch83000.parquet +rank0 [2026-11-15 08:43:17,171] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:43:17,284] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:43:17,354] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:43:17,422] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:43:17,490] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:43:17,559] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:43:17,628] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:43:17,739] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:43:17,847] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:43:17,952] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:43:18,022] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:43:18,089] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:43:18,155] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:43:18,221] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:43:18,287] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:43:18,390] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:43:18,398] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:43:18,400] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:43:18,403] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:43:18,405] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:43:18,408] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:43:18,411] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:43:18,413] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:43:18,416] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:43:18,418] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:43:18,420] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:43:18,423] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:43:18,425] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:43:18,428] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:43:18,430] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:43:18,433] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:43:26,134] INFO : VALIDATION | Step=83000/500000 | Train Loss=1.3647 | Valid Loss=2.6542 | Train Loss Unweighted=0.6794 | Valid Loss Unweighted=0.8275 | Stale=24 | ETA=1931.7m +rank0 [2026-11-15 08:43:26,134] INFO : split_configs=['1'] +rank0 [2026-11-15 08:43:26,148] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:43:26,155] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:43:26,273] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:43:27,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:43:27,701] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:43:28,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:43:28,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:43:28,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:43:29,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:43:29,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:43:29,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:43:30,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:43:30,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:43:30,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:43:31,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:43:31,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:43:31,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:43:32,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:43:32,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:43:32,562] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:43:32,565] INFO : split_configs=['1'] +rank0 [2026-11-15 08:43:32,583] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:43:32,586] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:43:32,676] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:43:33,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:43:34,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:43:34,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:43:35,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:43:35,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:43:36,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:43:36,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:43:37,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:43:37,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:43:38,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:43:38,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:43:38,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:43:39,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:43:39,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:43:40,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:43:40,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:43:40,794] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:43:40,797] INFO : split_configs=['1'] +rank0 [2026-11-15 08:43:40,813] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:43:40,817] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:43:40,907] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:43:42,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:43:42,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:43:42,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:43:43,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:43:43,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:43:44,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:43:44,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:43:44,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:43:45,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:43:45,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:43:46,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:43:46,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:43:47,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:43:47,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:43:47,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:43:48,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:43:48,387] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:43:48,394] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf +rank0 [2026-11-15 08:43:49,622] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf +rank0 [2026-11-15 08:43:50,039] INFO : Plotted number of elements +rank0 [2026-11-15 08:43:51,331] INFO : Plotted jets +rank0 [2026-11-15 08:43:52,362] INFO : Plotted jet ratio +rank0 [2026-11-15 08:43:52,373] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:43:53,999] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:43:54,462] INFO : Plotted number of elements +rank0 [2026-11-15 08:43:56,395] INFO : Plotted jets +rank0 [2026-11-15 08:43:57,423] INFO : Plotted jet ratio +rank0 [2026-11-15 08:43:57,432] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:43:58,926] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:43:59,370] INFO : Plotted number of elements +rank0 [2026-11-15 08:44:00,692] INFO : Plotted jets +rank0 [2026-11-15 08:44:01,680] INFO : Plotted jet ratio +rank0 [2026-11-15 08:47:55,539] INFO : Step 84000/500000 rank0 | Train Loss: 1.6607 | Train Loss Unweighted: 0.8552 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0068 | Regression_energy: 0.3005 | Regression_eta: 0.0173 | Regression_pt: 0.2906 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0885 | Classification_binary: 0.2340 | Regression_cos_phi: 0.4008 | Regression_energy: 0.1798 | Regression_eta: 0.2243 | Regression_pt: 0.1738 | Regression_sin_phi: 0.3595 | LR: 3.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4337s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0025 | max_forward: 0.0606 | max_loss: 0.0027 | max_backward: 0.1277 | max_optimizer_step: 0.0973 | max_tensorboard_logging: 1.7163 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11356.6 | uss_MB: 10212.5 | child_rss_MB: 23800.4 | allocated_MB: 196.4 | reserved_MB: 22190.0 | inactive_split_MB: 439.6 +rank0 [2026-11-15 08:47:55,542] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-84000.pth +rank0 [2026-11-15 08:47:55,612] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 08:47:55,626] INFO : Running validation on rank0 +rank0 [2026-11-15 08:47:55,626] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:47:55,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:47:56,251] INFO : saved batch0_epoch84000.parquet +rank0 [2026-11-15 08:47:56,373] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:47:56,487] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:47:56,558] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:47:56,625] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:47:56,692] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:47:56,758] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:47:56,825] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:47:56,932] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:47:57,037] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:47:57,139] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:47:57,208] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:47:57,274] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:47:57,341] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:47:57,409] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:47:57,478] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:47:57,585] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:47:57,606] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:47:57,618] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:47:57,621] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:47:57,630] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:47:57,644] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:47:57,647] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:47:57,657] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:47:57,660] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:47:57,663] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:47:57,665] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:47:57,668] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:47:57,670] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:47:57,673] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:47:57,676] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:47:57,678] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:48:05,366] INFO : VALIDATION | Step=84000/500000 | Train Loss=1.6607 | Valid Loss=2.7099 | Train Loss Unweighted=0.8552 | Valid Loss Unweighted=0.8406 | Stale=25 | ETA=1927.2m +rank0 [2026-11-15 08:48:05,367] INFO : split_configs=['1'] +rank0 [2026-11-15 08:48:05,383] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:48:05,393] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:48:05,517] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:48:06,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:48:06,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:48:07,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:48:07,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:48:08,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:48:08,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:48:08,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:48:09,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:48:09,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:48:09,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:48:10,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:48:10,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:48:10,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:48:11,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:48:11,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:48:11,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:48:11,856] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:48:11,860] INFO : split_configs=['1'] +rank0 [2026-11-15 08:48:11,878] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:48:11,881] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:48:11,970] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:48:13,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:48:13,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:48:14,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:48:14,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:48:15,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:48:15,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:48:15,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:48:16,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:48:16,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:48:17,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:48:17,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:48:18,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:48:18,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:48:19,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:48:19,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:48:19,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:48:20,102] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:48:20,106] INFO : split_configs=['1'] +rank0 [2026-11-15 08:48:20,122] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:48:20,125] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:48:20,217] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:48:21,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:48:21,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:48:22,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:48:22,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:48:23,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:48:23,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:48:23,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:48:24,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:48:24,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:48:25,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:48:25,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:48:25,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:48:26,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:48:26,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:48:27,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:48:27,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:48:27,743] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:48:27,751] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf +rank0 [2026-11-15 08:48:28,970] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf +rank0 [2026-11-15 08:48:29,399] INFO : Plotted number of elements +rank0 [2026-11-15 08:48:30,692] INFO : Plotted jets +rank0 [2026-11-15 08:48:31,735] INFO : Plotted jet ratio +rank0 [2026-11-15 08:48:31,747] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:48:33,378] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:48:33,819] INFO : Plotted number of elements +rank0 [2026-11-15 08:48:35,748] INFO : Plotted jets +rank0 [2026-11-15 08:48:36,795] INFO : Plotted jet ratio +rank0 [2026-11-15 08:48:36,806] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:48:38,258] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:48:38,696] INFO : Plotted number of elements +rank0 [2026-11-15 08:48:40,042] INFO : Plotted jets +rank0 [2026-11-15 08:48:41,027] INFO : Plotted jet ratio +rank0 [2026-11-15 08:48:41,527] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 08:48:41,527] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:52:36,546] INFO : Step 85000/500000 rank0 | Train Loss: 1.5298 | Train Loss Unweighted: 0.6989 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2461 | Regression_cos_phi: 0.0046 | Regression_energy: 0.2075 | Regression_eta: 0.0291 | Regression_pt: 0.2064 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.2461 | Regression_cos_phi: 0.2683 | Regression_energy: 0.1242 | Regression_eta: 0.3780 | Regression_pt: 0.1234 | Regression_sin_phi: 0.3108 | LR: 3.10e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5745s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0029 | max_forward: 0.0565 | max_loss: 0.0029 | max_backward: 0.1166 | max_optimizer_step: 0.0986 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 11453.0 | uss_MB: 10308.8 | child_rss_MB: 23721.8 | allocated_MB: 197.4 | reserved_MB: 18352.0 | inactive_split_MB: 236.6 +rank0 [2026-11-15 08:52:36,551] INFO : Running validation on rank0 +rank0 [2026-11-15 08:52:36,551] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:52:36,551] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:52:37,184] INFO : saved batch0_epoch85000.parquet +rank0 [2026-11-15 08:52:37,310] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:52:37,423] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:52:37,494] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:52:37,563] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:52:37,631] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:52:37,702] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:52:37,770] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:52:37,884] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:52:37,990] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:52:38,093] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:52:38,162] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:52:38,229] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:52:38,294] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:52:38,361] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:52:38,427] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:52:38,530] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:52:38,537] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:52:38,540] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:52:38,543] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:52:38,545] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:52:38,548] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:52:38,551] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:52:38,553] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:52:38,556] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:52:38,559] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:52:38,562] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:52:38,564] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:52:38,568] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:52:38,570] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:52:38,573] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:52:38,575] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:52:46,272] INFO : VALIDATION | Step=85000/500000 | Train Loss=1.5298 | Valid Loss=2.6408 | Train Loss Unweighted=0.6989 | Valid Loss Unweighted=0.8313 | Stale=26 | ETA=1922.8m +rank0 [2026-11-15 08:52:46,272] INFO : split_configs=['1'] +rank0 [2026-11-15 08:52:46,287] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:52:46,296] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:52:46,403] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:52:47,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:52:47,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:52:48,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:52:48,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:52:48,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:52:49,171] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:52:49,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:52:49,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:52:50,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:52:50,539] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:52:50,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:52:51,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:52:51,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:52:51,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:52:52,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:52:52,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:52:52,654] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:52:52,658] INFO : split_configs=['1'] +rank0 [2026-11-15 08:52:52,675] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:52:52,679] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:52:52,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:52:53,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:52:54,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:52:54,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:52:55,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:52:55,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:52:56,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:52:56,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:52:57,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:52:57,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:52:57,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:52:58,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:52:58,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:52:59,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:52:59,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:53:00,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:53:00,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:53:00,821] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:53:00,825] INFO : split_configs=['1'] +rank0 [2026-11-15 08:53:00,842] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:53:00,846] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:53:00,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:53:02,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:53:02,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:53:02,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:53:03,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:53:03,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:53:04,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:53:04,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:53:04,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:53:05,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:53:05,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:53:06,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:53:06,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:53:07,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:53:07,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:53:07,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:53:08,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:53:08,476] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:53:08,488] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf +rank0 [2026-11-15 08:53:09,741] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf +rank0 [2026-11-15 08:53:10,206] INFO : Plotted number of elements +rank0 [2026-11-15 08:53:11,514] INFO : Plotted jets +rank0 [2026-11-15 08:53:12,553] INFO : Plotted jet ratio +rank0 [2026-11-15 08:53:12,566] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:53:14,327] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:53:14,773] INFO : Plotted number of elements +rank0 [2026-11-15 08:53:16,736] INFO : Plotted jets +rank0 [2026-11-15 08:53:17,796] INFO : Plotted jet ratio +rank0 [2026-11-15 08:53:17,805] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:53:19,350] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:53:19,795] INFO : Plotted number of elements +rank0 [2026-11-15 08:53:21,150] INFO : Plotted jets +rank0 [2026-11-15 08:53:22,149] INFO : Plotted jet ratio +rank0 [2026-11-15 08:55:18,507] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 08:55:18,507] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:57:17,610] INFO : Step 86000/500000 rank0 | Train Loss: 1.1802 | Train Loss Unweighted: 0.6757 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2322 | Regression_cos_phi: 0.0039 | Regression_energy: 0.2151 | Regression_eta: 0.0125 | Regression_pt: 0.2080 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2322 | Regression_cos_phi: 0.2279 | Regression_energy: 0.1288 | Regression_eta: 0.1622 | Regression_pt: 0.1244 | Regression_sin_phi: 0.2289 | LR: 3.15e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.6251s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0672 | max_loss: 0.0030 | max_backward: 0.1431 | max_optimizer_step: 0.0995 | max_tensorboard_logging: 1.8721 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 11364.7 | uss_MB: 10220.6 | child_rss_MB: 23725.9 | allocated_MB: 197.5 | reserved_MB: 17468.0 | inactive_split_MB: 268.5 +rank0 [2026-11-15 08:57:17,612] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-86000.pth +rank0 [2026-11-15 08:57:17,683] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 08:57:17,698] INFO : Running validation on rank0 +rank0 [2026-11-15 08:57:17,698] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:57:17,698] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:57:18,315] INFO : saved batch0_epoch86000.parquet +rank0 [2026-11-15 08:57:18,442] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:57:18,565] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:57:18,639] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:57:18,708] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:57:18,777] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:57:18,846] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:57:18,916] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:57:19,029] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:57:19,135] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:57:19,238] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:57:19,306] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:57:19,372] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:57:19,439] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:57:19,506] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:57:19,572] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:57:19,675] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:57:19,695] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:57:19,708] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:57:19,711] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:57:19,721] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:57:19,733] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:57:19,737] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:57:19,743] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:57:19,746] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:57:19,748] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:57:19,750] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:57:19,754] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:57:19,757] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:57:19,759] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:57:19,762] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:57:19,764] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:57:27,436] INFO : VALIDATION | Step=86000/500000 | Train Loss=1.1802 | Valid Loss=2.6541 | Train Loss Unweighted=0.6757 | Valid Loss Unweighted=0.8425 | Stale=27 | ETA=1918.4m +rank0 [2026-11-15 08:57:27,436] INFO : split_configs=['1'] +rank0 [2026-11-15 08:57:27,457] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:57:27,465] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:57:27,580] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:57:28,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:57:29,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:57:29,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:57:29,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:57:30,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:57:30,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:57:30,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:57:31,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:57:31,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:57:31,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:57:32,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:57:32,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:57:32,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:57:33,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:57:33,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:57:33,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:57:33,965] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:57:33,968] INFO : split_configs=['1'] +rank0 [2026-11-15 08:57:33,986] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:57:33,990] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:57:34,080] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:57:35,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:57:35,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:57:36,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:57:36,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:57:37,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:57:37,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:57:37,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:57:38,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:57:38,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:57:39,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:57:39,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:57:40,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:57:40,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:57:41,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:57:41,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:57:41,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:57:42,120] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:57:42,123] INFO : split_configs=['1'] +rank0 [2026-11-15 08:57:42,141] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:57:42,144] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:57:42,234] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:57:43,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:57:43,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:57:44,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:57:44,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:57:45,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:57:45,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:57:45,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:57:46,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:57:46,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:57:47,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:57:47,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:57:47,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:57:48,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:57:48,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:57:49,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:57:49,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:57:49,758] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:57:49,767] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf +rank0 [2026-11-15 08:57:51,003] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf +rank0 [2026-11-15 08:57:51,433] INFO : Plotted number of elements +rank0 [2026-11-15 08:57:52,742] INFO : Plotted jets +rank0 [2026-11-15 08:57:53,766] INFO : Plotted jet ratio +rank0 [2026-11-15 08:57:53,791] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:57:55,513] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:57:55,959] INFO : Plotted number of elements +rank0 [2026-11-15 08:57:57,913] INFO : Plotted jets +rank0 [2026-11-15 08:57:58,966] INFO : Plotted jet ratio +rank0 [2026-11-15 08:57:58,981] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:58:00,471] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:58:00,919] INFO : Plotted number of elements +rank0 [2026-11-15 08:58:02,269] INFO : Plotted jets +rank0 [2026-11-15 08:58:03,259] INFO : Plotted jet ratio +rank0 [2026-11-15 09:01:58,239] INFO : Step 87000/500000 rank0 | Train Loss: 1.3234 | Train Loss Unweighted: 0.6338 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1992 | Regression_eta: 0.0255 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0833 | Classification_binary: 0.2101 | Regression_cos_phi: 0.1993 | Regression_energy: 0.1192 | Regression_eta: 0.3309 | Regression_pt: 0.1143 | Regression_sin_phi: 0.2663 | LR: 3.20e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.7336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0622 | max_loss: 0.0028 | max_backward: 0.1322 | max_optimizer_step: 0.0997 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 11484.2 | uss_MB: 10340.1 | child_rss_MB: 23794.0 | allocated_MB: 196.0 | reserved_MB: 21958.0 | inactive_split_MB: 226.0 +rank0 [2026-11-15 09:01:58,242] INFO : Running validation on rank0 +rank0 [2026-11-15 09:01:58,242] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 09:01:58,242] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 09:01:58,880] INFO : saved batch0_epoch87000.parquet +rank0 [2026-11-15 09:01:59,003] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 09:01:59,120] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 09:01:59,190] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 09:01:59,258] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 09:01:59,324] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 09:01:59,392] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 09:01:59,459] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 09:01:59,570] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 09:01:59,674] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 09:01:59,776] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 09:01:59,844] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 09:01:59,911] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 09:01:59,976] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 09:02:00,044] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 09:02:00,115] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 09:02:00,222] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 09:02:00,241] INFO : plotted pt_target histogram +rank0 [2026-11-15 09:02:00,254] INFO : plotted pt_pred histogram +rank0 [2026-11-15 09:02:00,257] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 09:02:00,272] INFO : plotted eta_target histogram +rank0 [2026-11-15 09:02:00,275] INFO : plotted eta_pred histogram +rank0 [2026-11-15 09:02:00,277] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 09:02:00,281] INFO : plotted sphi_target histogram +rank0 [2026-11-15 09:02:00,297] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 09:02:00,299] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 09:02:00,301] INFO : plotted cphi_target histogram +rank0 [2026-11-15 09:02:00,304] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 09:02:00,306] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 09:02:00,308] INFO : plotted energy_target histogram +rank0 [2026-11-15 09:02:00,311] INFO : plotted energy_pred histogram +rank0 [2026-11-15 09:02:00,313] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 09:02:08,033] INFO : VALIDATION | Step=87000/500000 | Train Loss=1.3234 | Valid Loss=2.6329 | Train Loss Unweighted=0.6338 | Valid Loss Unweighted=0.8365 | Stale=28 | ETA=1914.0m +rank0 [2026-11-15 09:02:08,033] INFO : split_configs=['1'] +rank0 [2026-11-15 09:02:08,050] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 09:02:08,057] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 09:02:08,178] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:02:09,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 09:02:09,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 09:02:10,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 09:02:10,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 09:02:10,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 09:02:11,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 09:02:11,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 09:02:11,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 09:02:12,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 09:02:12,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 09:02:12,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 09:02:13,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 09:02:13,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 09:02:13,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 09:02:14,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 09:02:14,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 09:02:14,616] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 09:02:14,619] INFO : split_configs=['1'] +rank0 [2026-11-15 09:02:14,637] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 09:02:14,641] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 09:02:14,731] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:02:16,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 09:02:16,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 09:02:16,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 09:02:17,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 09:02:17,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 09:02:18,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 09:02:18,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 09:02:19,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 09:02:19,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 09:02:20,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 09:02:20,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 09:02:20,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 09:02:21,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 09:02:21,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 09:02:22,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 09:02:22,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 09:02:22,869] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 09:02:22,873] INFO : split_configs=['1'] +rank0 [2026-11-15 09:02:22,890] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 09:02:22,894] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:02:22,986] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:02:24,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 09:02:24,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 09:02:25,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 09:02:25,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 09:02:25,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 09:02:26,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 09:02:26,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 09:02:27,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 09:02:27,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 09:02:27,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 09:02:28,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 09:02:28,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 09:02:29,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 09:02:29,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 09:02:29,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 09:02:30,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 09:02:30,538] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 09:02:30,547] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf +rank0 [2026-11-15 09:02:31,760] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf +rank0 [2026-11-15 09:02:32,186] INFO : Plotted number of elements +rank0 [2026-11-15 09:02:33,479] INFO : Plotted jets +rank0 [2026-11-15 09:02:34,507] INFO : Plotted jet ratio +rank0 [2026-11-15 09:02:34,520] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:02:36,177] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:02:36,639] INFO : Plotted number of elements +rank0 [2026-11-15 09:02:38,559] INFO : Plotted jets +rank0 [2026-11-15 09:02:39,586] INFO : Plotted jet ratio +rank0 [2026-11-15 09:02:39,595] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:02:41,099] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:02:41,540] INFO : Plotted number of elements +rank0 [2026-11-15 09:02:42,851] INFO : Plotted jets +rank0 [2026-11-15 09:02:43,840] INFO : Plotted jet ratio +rank0 [2026-11-15 09:02:44,344] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 09:02:44,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 09:06:39,306] INFO : Step 88000/500000 rank0 | Train Loss: 1.2959 | Train Loss Unweighted: 0.7170 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0033 | Regression_energy: 0.2280 | Regression_eta: 0.0200 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2387 | Regression_cos_phi: 0.1959 | Regression_energy: 0.1364 | Regression_eta: 0.2604 | Regression_pt: 0.1331 | Regression_sin_phi: 0.2634 | LR: 3.25e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.7512s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0023 | max_forward: 0.0585 | max_loss: 0.0028 | max_backward: 0.1243 | max_optimizer_step: 0.0999 | max_tensorboard_logging: 1.9806 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11651.0 | uss_MB: 10506.8 | child_rss_MB: 23863.8 | allocated_MB: 193.3 | reserved_MB: 18734.0 | inactive_split_MB: 192.7 +rank0 [2026-11-15 09:06:39,309] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-88000.pth +rank0 [2026-11-15 09:06:39,382] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 09:06:39,385] INFO : Running validation on rank0 +rank0 [2026-11-15 09:06:39,385] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 09:06:39,385] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 09:06:40,017] INFO : saved batch0_epoch88000.parquet +rank0 [2026-11-15 09:06:40,147] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 09:06:40,262] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 09:06:40,333] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 09:06:40,401] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 09:06:40,467] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 09:06:40,534] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 09:06:40,602] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 09:06:40,712] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 09:06:40,818] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 09:06:40,924] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 09:06:40,995] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 09:06:41,064] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 09:06:41,134] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 09:06:41,205] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 09:06:41,274] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 09:06:41,380] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 09:06:41,387] INFO : plotted pt_target histogram +rank0 [2026-11-15 09:06:41,390] INFO : plotted pt_pred histogram +rank0 [2026-11-15 09:06:41,392] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 09:06:41,395] INFO : plotted eta_target histogram +rank0 [2026-11-15 09:06:41,398] INFO : plotted eta_pred histogram +rank0 [2026-11-15 09:06:41,400] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 09:06:41,402] INFO : plotted sphi_target histogram +rank0 [2026-11-15 09:06:41,405] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 09:06:41,407] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 09:06:41,410] INFO : plotted cphi_target histogram +rank0 [2026-11-15 09:06:41,412] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 09:06:41,415] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 09:06:41,417] INFO : plotted energy_target histogram +rank0 [2026-11-15 09:06:41,420] INFO : plotted energy_pred histogram +rank0 [2026-11-15 09:06:41,422] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 09:06:49,165] INFO : VALIDATION | Step=88000/500000 | Train Loss=1.2959 | Valid Loss=2.6176 | Train Loss Unweighted=0.7170 | Valid Loss Unweighted=0.8337 | Stale=29 | ETA=1909.6m +rank0 [2026-11-15 09:06:49,165] INFO : split_configs=['1'] +rank0 [2026-11-15 09:06:49,180] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 09:06:49,190] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 09:06:49,299] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:06:50,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 09:06:50,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 09:06:51,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 09:06:51,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 09:06:51,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 09:06:52,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 09:06:52,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 09:06:52,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 09:06:53,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 09:06:53,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 09:06:53,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 09:06:54,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 09:06:54,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 09:06:54,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 09:06:55,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 09:06:55,461] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 09:06:55,580] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 09:06:55,583] INFO : split_configs=['1'] +rank0 [2026-11-15 09:06:55,599] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 09:06:55,603] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 09:06:55,687] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:06:56,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 09:06:57,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 09:06:57,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 09:06:58,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 09:06:58,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 09:06:59,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 09:06:59,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 09:06:59,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 09:07:00,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 09:07:00,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 09:07:01,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 09:07:01,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 09:07:02,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 09:07:02,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 09:07:03,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 09:07:03,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 09:07:03,714] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 09:07:03,718] INFO : split_configs=['1'] +rank0 [2026-11-15 09:07:03,735] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 09:07:03,738] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:07:03,826] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:07:05,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 09:07:05,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 09:07:05,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 09:07:06,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 09:07:06,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 09:07:07,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 09:07:07,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 09:07:07,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 09:07:08,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 09:07:08,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 09:07:09,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 09:07:09,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 09:07:09,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 09:07:10,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 09:07:10,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 09:07:11,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 09:07:11,281] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 09:07:11,290] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf +rank0 [2026-11-15 09:07:12,537] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf +rank0 [2026-11-15 09:07:12,976] INFO : Plotted number of elements +rank0 [2026-11-15 09:07:14,286] INFO : Plotted jets +rank0 [2026-11-15 09:07:15,311] INFO : Plotted jet ratio +rank0 [2026-11-15 09:07:15,323] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:07:17,062] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:07:17,506] INFO : Plotted number of elements +rank0 [2026-11-15 09:07:19,471] INFO : Plotted jets +rank0 [2026-11-15 09:07:20,523] INFO : Plotted jet ratio +rank0 [2026-11-15 09:07:20,533] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:07:22,021] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:07:22,468] INFO : Plotted number of elements +rank0 [2026-11-15 09:07:23,819] INFO : Plotted jets +rank0 [2026-11-15 09:07:24,821] INFO : Plotted jet ratio +rank0 [2026-11-15 09:09:21,252] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 09:09:21,252] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 09:11:19,970] INFO : Step 89000/500000 rank0 | Train Loss: 1.2001 | Train Loss Unweighted: 0.7272 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2398 | Regression_cos_phi: 0.0048 | Regression_energy: 0.2359 | Regression_eta: 0.0094 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.2398 | Regression_cos_phi: 0.2792 | Regression_energy: 0.1412 | Regression_eta: 0.1224 | Regression_pt: 0.1398 | Regression_sin_phi: 0.2014 | LR: 3.29e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5640s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0700 | max_loss: 0.0032 | max_backward: 0.1516 | max_optimizer_step: 0.1023 | max_tensorboard_logging: 1.7682 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11622.6 | uss_MB: 10478.5 | child_rss_MB: 23648.5 | allocated_MB: 197.4 | reserved_MB: 23364.0 | inactive_split_MB: 496.6 +rank0 [2026-11-15 09:11:19,975] INFO : Running validation on rank0 +rank0 [2026-11-15 09:11:19,976] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 09:11:19,976] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 09:11:20,608] INFO : saved batch0_epoch89000.parquet +rank0 [2026-11-15 09:11:20,735] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 09:11:20,851] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 09:11:20,922] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 09:11:20,991] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 09:11:21,061] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 09:11:21,130] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 09:11:21,199] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 09:11:21,313] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 09:11:21,420] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 09:11:21,527] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 09:11:21,598] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 09:11:21,669] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 09:11:21,738] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 09:11:21,807] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 09:11:21,874] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 09:11:21,979] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 09:11:21,987] INFO : plotted pt_target histogram +rank0 [2026-11-15 09:11:21,989] INFO : plotted pt_pred histogram +rank0 [2026-11-15 09:11:21,992] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 09:11:21,995] INFO : plotted eta_target histogram +rank0 [2026-11-15 09:11:21,997] INFO : plotted eta_pred histogram +rank0 [2026-11-15 09:11:22,000] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 09:11:22,002] INFO : plotted sphi_target histogram +rank0 [2026-11-15 09:11:22,005] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 09:11:22,007] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 09:11:22,010] INFO : plotted cphi_target histogram +rank0 [2026-11-15 09:11:22,013] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 09:11:22,015] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 09:11:22,018] INFO : plotted energy_target histogram +rank0 [2026-11-15 09:11:22,020] INFO : plotted energy_pred histogram +rank0 [2026-11-15 09:11:22,023] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 09:11:29,749] INFO : VALIDATION | Step=89000/500000 | Train Loss=1.2001 | Valid Loss=2.6601 | Train Loss Unweighted=0.7272 | Valid Loss Unweighted=0.8352 | Stale=30 | ETA=1905.2m +rank0 [2026-11-15 09:11:29,749] INFO : split_configs=['1'] +rank0 [2026-11-15 09:11:29,765] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 09:11:29,773] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 09:11:29,912] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:11:31,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 09:11:31,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 09:11:31,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 09:11:32,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 09:11:32,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 09:11:32,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 09:11:33,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 09:11:33,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 09:11:33,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 09:11:34,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 09:11:34,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 09:11:34,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 09:11:35,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 09:11:35,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 09:11:35,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 09:11:36,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 09:11:36,275] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 09:11:36,278] INFO : split_configs=['1'] +rank0 [2026-11-15 09:11:36,296] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 09:11:36,300] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 09:11:36,391] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:11:37,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 09:11:38,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 09:11:38,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 09:11:38,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 09:11:39,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 09:11:39,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 09:11:40,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 09:11:40,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 09:11:41,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 09:11:41,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 09:11:42,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 09:11:42,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 09:11:43,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 09:11:43,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 09:11:43,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 09:11:44,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 09:11:44,475] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 09:11:44,478] INFO : split_configs=['1'] +rank0 [2026-11-15 09:11:44,496] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 09:11:44,501] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:11:44,594] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:11:45,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 09:11:46,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 09:11:46,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 09:11:47,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 09:11:47,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 09:11:47,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 09:11:48,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 09:11:48,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 09:11:49,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 09:11:49,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 09:11:49,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 09:11:50,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 09:11:50,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 09:11:51,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 09:11:51,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 09:11:52,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 09:11:52,169] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 09:11:52,177] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf +rank0 [2026-11-15 09:11:53,428] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf +rank0 [2026-11-15 09:11:53,866] INFO : Plotted number of elements +rank0 [2026-11-15 09:11:55,169] INFO : Plotted jets +rank0 [2026-11-15 09:11:56,187] INFO : Plotted jet ratio +rank0 [2026-11-15 09:11:56,198] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:11:57,875] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:11:58,342] INFO : Plotted number of elements +rank0 [2026-11-15 09:12:00,336] INFO : Plotted jets +rank0 [2026-11-15 09:12:01,379] INFO : Plotted jet ratio +rank0 [2026-11-15 09:12:01,389] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:12:02,859] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:12:03,305] INFO : Plotted number of elements +rank0 [2026-11-15 09:12:04,647] INFO : Plotted jets +rank0 [2026-11-15 09:12:05,640] INFO : Plotted jet ratio +rank0 [2026-11-15 09:12:06,156] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 09:12:06,157] INFO : Training completed. Total time on device 0: 413.163min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..f00db2d78a3c46cd7460f09217d19f588188861d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/train.log.0 @@ -0,0 +1,10818 @@ +rank0 [2026-11-15 02:18:53,736] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 02:18:53,736] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 02:18:53,737] INFO : Instantiating model +rank0 [2026-11-15 02:18:53,774] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 02:18:53,775] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 02:18:53,776] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 02:18:53,776] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 02:18:53,776] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 02:18:53,776] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 02:18:53,777] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,779] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 02:18:53,779] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,782] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 02:18:53,782] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,784] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 02:18:53,785] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,787] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 02:18:53,787] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,790] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 02:18:53,790] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,792] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 02:18:53,793] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 02:18:53,793] INFO : Initializing output DNNs +rank0 [2026-11-15 02:18:53,807] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,810] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 02:18:53,817] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-15 02:18:53,817] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 02:18:53,817] INFO : jagged_attention enabled=True +rank0 [2026-11-15 02:18:53,817] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 02:18:53,817] INFO : task_queries enabled=True +rank0 [2026-11-15 02:18:53,817] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 02:18:53,817] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 02:18:53,817] INFO : classification_query parameters: 256 +rank0 [2026-11-15 02:18:53,817] INFO : regression_query parameters: 256 +rank0 [2026-11-15 02:18:53,817] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 02:18:53,817] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 02:18:53,817] INFO : output_mode=elementwise +rank0 [2026-11-15 02:18:53,817] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 02:18:53,817] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 02:18:53,817] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 02:18:53,817] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 02:18:53,818] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 02:18:53,818] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 02:18:53,818] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 02:18:53,818] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 02:18:53,819] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 02:18:53,819] INFO : MLPF __init__ done +rank0 [2026-11-15 02:18:53,819] INFO : Instantiated model +rank0 [2026-11-15 02:18:53,820] INFO : Moving model to device rank=0 +rank0 [2026-11-15 02:18:53,831] INFO : Moved model to device rank=0 +rank0 [2026-11-15 02:18:53,832] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 02:18:53,963] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 02:18:53,963] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 02:18:53,963] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 02:18:54,062] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 02:18:54,062] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 02:18:54,064] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 02:18:54,064] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 02:18:54,064] INFO : Total parameters: 3910930 +rank0 [2026-11-15 02:18:54,066] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 02:18:54,071] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856 +rank0 [2026-11-15 02:18:54,071] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856 +rank0 [2026-11-15 02:18:54,071] INFO : Getting dataloaders +rank0 [2026-11-15 02:18:54,072] INFO : sample=idea_edm_qq_pf split=train split_configs=['1'] +rank0 [2026-11-15 02:18:56,127] INFO : train_dataset: idea_edm_qq_pf, 8000 +rank0 [2026-11-15 02:18:56,127] INFO : sample=idea_edm_ttbar_pf split=train split_configs=['1'] +rank0 [2026-11-15 02:18:56,171] INFO : train_dataset: idea_edm_ttbar_pf, 8000 +rank0 [2026-11-15 02:18:56,171] INFO : sample=idea_edm_ww_fullhad_pf split=train split_configs=['1'] +rank0 [2026-11-15 02:18:56,214] INFO : train_dataset: idea_edm_ww_fullhad_pf, 8000 +rank0 [2026-11-15 02:18:56,215] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 02:18:56,215] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 02:18:56,215] INFO : train mix: physical: weight=1500 step_share=1.000 batches_per_epoch=1500 events_per_step=16 expected_passes_in_500000_steps=333.33 +rank0 [2026-11-15 02:18:56,215] INFO : sample=idea_edm_qq_pf split=valid split_configs=['1'] +rank0 [2026-11-15 02:18:56,223] INFO : valid_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:18:56,223] INFO : sample=idea_edm_ttbar_pf split=valid split_configs=['1'] +rank0 [2026-11-15 02:18:56,231] INFO : valid_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:18:56,231] INFO : sample=idea_edm_ww_fullhad_pf split=valid split_configs=['1'] +rank0 [2026-11-15 02:18:56,240] INFO : valid_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:18:56,240] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 02:18:56,240] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 02:18:56,294] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:18:56,382] INFO : Got dataloaders +rank0 [2026-11-15 02:18:56,383] INFO : loader split=train rank=0 len=1500 +rank0 [2026-11-15 02:18:56,383] INFO : loader split=valid rank=0 len=48 +rank0 [2026-11-15 02:18:56,383] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 02:18:56,392] INFO : Creating train_iterator +rank0 [2026-11-15 02:18:56,392] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:18:56,412] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:18:56,519] INFO : Created train_iterator +rank0 [2026-11-15 02:19:21,499] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9513028985261918, 'Classification': 0.007238105600699782, 'Regression_pt': 3.262555984258652, 'Regression_eta': 0.15020455058664084, 'Regression_sin_phi': 0.031022030971944333, 'Regression_cos_phi': 0.03321564427576959, 'Regression_energy': 3.2605306005477903}, weights={'Classification_binary': 1.0, 'Classification': 269.5875244140625, 'Regression_pt': 0.5980902314186096, 'Regression_eta': 12.990970611572266, 'Regression_sin_phi': 62.90055465698242, 'Regression_cos_phi': 58.74650192260742, 'Regression_energy': 0.5984617471694946} +rank0 [2026-11-15 02:22:52,456] INFO : Step 1000/500000 rank0 | Train Loss: 4.6716 | Train Loss Unweighted: 1.8902 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2859 | Regression_cos_phi: 0.0147 | Regression_energy: 0.7599 | Regression_eta: 0.0895 | Regression_pt: 0.7188 | Regression_sin_phi: 0.0208 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1668 | Classification_binary: 0.2859 | Regression_cos_phi: 0.8650 | Regression_energy: 0.4548 | Regression_eta: 1.1624 | Regression_pt: 0.4299 | Regression_sin_phi: 1.3069 | LR: 2.01e-05 | DataLoad Time: 0.0018s | Model Forward Time: 1.2127s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0026 | max_forward: 0.0682 | max_loss: 0.0030 | max_backward: 0.1429 | max_optimizer_step: 0.0569 | max_tensorboard_logging: 0.7698 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.6 | uss_MB: 1351.5 | child_rss_MB: 23464.4 | allocated_MB: 199.6 | reserved_MB: 23934.0 | inactive_split_MB: 486.4 +rank0 [2026-11-15 02:22:52,457] INFO : Running validation on rank0 +rank0 [2026-11-15 02:22:52,458] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:22:52,458] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:22:53,109] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 02:22:53,269] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:22:53,347] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:22:53,401] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:22:53,455] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:22:53,501] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:22:53,555] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:22:53,611] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:22:53,659] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:22:53,727] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:22:53,795] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:22:53,850] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:22:53,905] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:22:53,953] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:22:54,008] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:22:54,063] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:22:54,111] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:22:54,116] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:22:54,119] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:22:54,121] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:22:54,124] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:22:54,127] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:22:54,129] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:22:54,132] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:22:54,156] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:22:54,159] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:22:54,211] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:22:54,214] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:22:54,216] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:22:54,219] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:22:54,222] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:22:54,224] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:23:01,883] INFO : VALIDATION | Step=1000/500000 | Train Loss=4.6716 | Valid Loss=4.2833 | Train Loss Unweighted=1.8902 | Valid Loss Unweighted=1.5921 | Stale=0 | ETA=2041.7m +rank0 [2026-11-15 02:23:01,883] INFO : split_configs=['1'] +rank0 [2026-11-15 02:23:01,895] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:23:01,901] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:23:01,964] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:02,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:03,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:03,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:03,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:04,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:04,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:04,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:05,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:05,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:05,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:06,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:06,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:06,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:07,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:07,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:07,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:07,820] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 02:23:07,822] INFO : split_configs=['1'] +rank0 [2026-11-15 02:23:07,833] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:23:07,837] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:23:07,873] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:08,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:09,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:09,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:10,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:10,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:10,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:11,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:11,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:12,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:12,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:13,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:13,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:13,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:14,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:14,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:15,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:15,324] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:23:15,327] INFO : split_configs=['1'] +rank0 [2026-11-15 02:23:15,337] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:23:15,342] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:23:15,374] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:23:16,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:23:16,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:23:16,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:23:17,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:23:17,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:23:18,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:23:18,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:23:19,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:23:19,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:23:19,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:23:20,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:23:20,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:23:21,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:23:21,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:23:21,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:23:22,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:23:22,350] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:23:22,358] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf +rank0 [2026-11-15 02:23:23,488] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_qq_pf +rank0 [2026-11-15 02:23:23,977] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:25,650] INFO : Plotted jets +rank0 [2026-11-15 02:23:26,666] INFO : Plotted jet ratio +rank0 [2026-11-15 02:23:26,678] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:23:28,262] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:23:28,703] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:30,052] INFO : Plotted jets +rank0 [2026-11-15 02:23:31,077] INFO : Plotted jet ratio +rank0 [2026-11-15 02:23:31,090] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:23:32,492] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_1000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:23:32,931] INFO : Plotted number of elements +rank0 [2026-11-15 02:23:34,241] INFO : Plotted jets +rank0 [2026-11-15 02:23:35,220] INFO : Plotted jet ratio +rank0 [2026-11-15 02:25:31,934] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 02:25:31,934] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:27:28,686] INFO : Step 2000/500000 rank0 | Train Loss: 4.2064 | Train Loss Unweighted: 1.3477 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2688 | Regression_cos_phi: 0.0168 | Regression_energy: 0.4710 | Regression_eta: 0.1028 | Regression_pt: 0.4738 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1751 | Classification_binary: 0.2688 | Regression_cos_phi: 0.9847 | Regression_energy: 0.2818 | Regression_eta: 1.3358 | Regression_pt: 0.2834 | Regression_sin_phi: 0.8768 | LR: 2.02e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.2354s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0690 | max_loss: 0.0029 | max_backward: 0.1503 | max_optimizer_step: 0.0604 | max_tensorboard_logging: 0.7507 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 4841.1 | uss_MB: 3619.9 | child_rss_MB: 23751.4 | allocated_MB: 197.5 | reserved_MB: 19548.0 | inactive_split_MB: 136.5 +rank0 [2026-11-15 02:27:28,688] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 02:27:28,752] INFO : Running validation on rank0 +rank0 [2026-11-15 02:27:28,752] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:27:28,752] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:27:29,363] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 02:27:29,481] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:27:29,593] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:27:29,660] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:27:29,725] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:27:29,789] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:27:29,854] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:27:29,918] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:27:30,023] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:27:30,125] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:27:30,224] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:27:30,290] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:27:30,354] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:27:30,417] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:27:30,482] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:27:30,546] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:27:30,645] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:27:30,664] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:27:30,677] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:27:30,680] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:27:30,690] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:27:30,703] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:27:30,706] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:27:30,721] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:27:30,724] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:27:30,726] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:27:30,728] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:27:30,732] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:27:30,747] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:27:30,749] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:27:30,752] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:27:30,754] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:27:38,443] INFO : VALIDATION | Step=2000/500000 | Train Loss=4.2064 | Valid Loss=3.8027 | Train Loss Unweighted=1.3477 | Valid Loss Unweighted=1.3612 | Stale=0 | ETA=2166.5m +rank0 [2026-11-15 02:27:38,443] INFO : split_configs=['1'] +rank0 [2026-11-15 02:27:38,454] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:27:38,461] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:27:38,528] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:39,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:39,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:39,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:40,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:40,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:40,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:41,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:41,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:41,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:42,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:42,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:42,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:43,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:43,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:43,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:44,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:44,338] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:27:44,341] INFO : split_configs=['1'] +rank0 [2026-11-15 02:27:44,351] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:27:44,355] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:27:44,391] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:45,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:45,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:46,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:46,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:46,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:47,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:47,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:48,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:48,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:49,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:49,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:50,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:50,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:50,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:51,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:51,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:51,897] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:27:51,901] INFO : split_configs=['1'] +rank0 [2026-11-15 02:27:51,912] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:27:51,916] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:27:51,954] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:27:52,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:27:53,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:27:53,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:27:53,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:27:54,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:27:54,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:27:55,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:27:55,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:27:56,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:27:56,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:27:56,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:27:57,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:27:57,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:27:58,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:27:58,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:27:58,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:27:58,948] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:27:58,956] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf +rank0 [2026-11-15 02:28:00,019] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_qq_pf +rank0 [2026-11-15 02:28:00,453] INFO : Plotted number of elements +rank0 [2026-11-15 02:28:02,127] INFO : Plotted jets +rank0 [2026-11-15 02:28:03,143] INFO : Plotted jet ratio +rank0 [2026-11-15 02:28:03,156] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:28:04,663] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:28:05,117] INFO : Plotted number of elements +rank0 [2026-11-15 02:28:06,435] INFO : Plotted jets +rank0 [2026-11-15 02:28:07,442] INFO : Plotted jet ratio +rank0 [2026-11-15 02:28:07,452] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:28:08,858] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_2000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:28:09,317] INFO : Plotted number of elements +rank0 [2026-11-15 02:28:10,609] INFO : Plotted jets +rank0 [2026-11-15 02:28:11,593] INFO : Plotted jet ratio +rank0 [2026-11-15 02:32:06,351] INFO : Step 3000/500000 rank0 | Train Loss: 5.3811 | Train Loss Unweighted: 1.5084 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2532 | Regression_cos_phi: 0.0121 | Regression_energy: 0.5437 | Regression_eta: 0.2140 | Regression_pt: 0.4705 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1254 | Classification_binary: 0.2532 | Regression_cos_phi: 0.7113 | Regression_energy: 0.3254 | Regression_eta: 2.7795 | Regression_pt: 0.2814 | Regression_sin_phi: 0.9049 | LR: 2.05e-05 | DataLoad Time: 0.0015s | Model Forward Time: 1.2663s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0614 | max_loss: 0.0026 | max_backward: 0.1330 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.7979 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5418.4 | uss_MB: 4197.4 | child_rss_MB: 23862.1 | allocated_MB: 198.0 | reserved_MB: 16382.0 | inactive_split_MB: 172.0 +rank0 [2026-11-15 02:32:06,356] INFO : Running validation on rank0 +rank0 [2026-11-15 02:32:06,356] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:32:06,356] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:32:06,976] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 02:32:07,095] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:32:07,211] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:32:07,280] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:32:07,345] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:32:07,411] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:32:07,477] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:32:07,543] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:32:07,649] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:32:07,751] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:32:07,851] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:32:07,919] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:32:07,984] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:32:08,049] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:32:08,116] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:32:08,181] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:32:08,283] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:32:08,306] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:32:08,319] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:32:08,322] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:32:08,333] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:32:08,365] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:32:08,382] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:32:08,383] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:32:08,385] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:32:08,386] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:32:08,387] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:32:08,396] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:32:08,397] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:32:08,398] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:32:08,399] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:32:08,400] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:32:16,091] INFO : VALIDATION | Step=3000/500000 | Train Loss=5.3811 | Valid Loss=3.5861 | Train Loss Unweighted=1.5084 | Valid Loss Unweighted=1.2585 | Stale=0 | ETA=2208.1m +rank0 [2026-11-15 02:32:16,092] INFO : split_configs=['1'] +rank0 [2026-11-15 02:32:16,103] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:32:16,116] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:32:16,182] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:32:16,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:32:17,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:32:17,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:32:17,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:32:18,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:32:18,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:32:18,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:32:19,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:32:19,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:32:19,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:32:20,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:32:20,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:32:20,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:32:21,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:32:21,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:32:21,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:32:21,942] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:32:21,945] INFO : split_configs=['1'] +rank0 [2026-11-15 02:32:21,956] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:32:21,960] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:32:22,000] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:32:22,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:32:23,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:32:23,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:32:24,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:32:24,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:32:25,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:32:25,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:32:25,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:32:26,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:32:26,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:32:27,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:32:27,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:32:28,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:32:28,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:32:29,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:32:29,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:32:29,567] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:32:29,571] INFO : split_configs=['1'] +rank0 [2026-11-15 02:32:29,582] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:32:29,586] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:32:29,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:32:30,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:32:30,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:32:31,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:32:31,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:32:32,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:32:32,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:32:32,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:32:33,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:32:33,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:32:34,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:32:34,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:32:34,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:32:35,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:32:35,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:32:36,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:32:36,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:32:36,587] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:32:36,595] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf +rank0 [2026-11-15 02:32:37,689] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_qq_pf +rank0 [2026-11-15 02:32:38,127] INFO : Plotted number of elements +rank0 [2026-11-15 02:32:39,800] INFO : Plotted jets +rank0 [2026-11-15 02:32:40,829] INFO : Plotted jet ratio +rank0 [2026-11-15 02:32:40,842] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:32:42,356] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:32:42,795] INFO : Plotted number of elements +rank0 [2026-11-15 02:32:44,109] INFO : Plotted jets +rank0 [2026-11-15 02:32:45,125] INFO : Plotted jet ratio +rank0 [2026-11-15 02:32:45,138] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:32:46,512] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_3000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:32:46,944] INFO : Plotted number of elements +rank0 [2026-11-15 02:32:48,243] INFO : Plotted jets +rank0 [2026-11-15 02:32:49,216] INFO : Plotted jet ratio +rank0 [2026-11-15 02:32:49,585] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 02:32:49,585] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:36:43,273] INFO : Step 4000/500000 rank0 | Train Loss: 3.2529 | Train Loss Unweighted: 1.2183 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2756 | Regression_cos_phi: 0.0109 | Regression_energy: 0.4325 | Regression_eta: 0.0706 | Regression_pt: 0.4162 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1572 | Classification_binary: 0.2756 | Regression_cos_phi: 0.6375 | Regression_energy: 0.2588 | Regression_eta: 0.9176 | Regression_pt: 0.2489 | Regression_sin_phi: 0.7573 | LR: 2.08e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.3199s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0026 | max_forward: 0.0650 | max_loss: 0.0029 | max_backward: 0.1395 | max_optimizer_step: 0.0639 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0006 | Memory: rss_MB: 5603.4 | uss_MB: 4382.4 | child_rss_MB: 23919.4 | allocated_MB: 200.6 | reserved_MB: 19108.0 | inactive_split_MB: 225.4 +rank0 [2026-11-15 02:36:43,275] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 02:36:43,341] INFO : Running validation on rank0 +rank0 [2026-11-15 02:36:43,341] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:36:43,341] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:36:43,961] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 02:36:44,080] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:36:44,195] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:36:44,266] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:36:44,332] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:36:44,398] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:36:44,467] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:36:44,534] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:36:44,641] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:36:44,745] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:36:44,848] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:36:44,916] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:36:44,983] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:36:45,051] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:36:45,119] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:36:45,186] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:36:45,289] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:36:45,311] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:36:45,324] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:36:45,325] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:36:45,362] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:36:45,375] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:36:45,378] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:36:45,381] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:36:45,386] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:36:45,389] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:36:45,392] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:36:45,395] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:36:45,398] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:36:45,402] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:36:45,405] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:36:45,408] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:36:53,103] INFO : VALIDATION | Step=4000/500000 | Train Loss=3.2529 | Valid Loss=3.4666 | Train Loss Unweighted=1.2183 | Valid Loss Unweighted=1.2061 | Stale=0 | ETA=2225.2m +rank0 [2026-11-15 02:36:53,104] INFO : split_configs=['1'] +rank0 [2026-11-15 02:36:53,115] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:36:53,119] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:36:53,190] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:36:53,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:36:54,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:36:54,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:36:54,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:36:55,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:36:55,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:36:56,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:36:56,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:36:56,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:36:57,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:36:57,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:36:57,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:36:58,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:36:58,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:36:58,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:36:59,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:36:59,086] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:36:59,088] INFO : split_configs=['1'] +rank0 [2026-11-15 02:36:59,099] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:36:59,114] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:36:59,156] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:37:00,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:37:00,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:37:00,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:37:01,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:37:01,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:37:02,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:37:02,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:37:03,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:37:03,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:37:04,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:37:04,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:37:04,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:37:05,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:37:05,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:37:06,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:37:06,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:37:06,749] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:37:06,752] INFO : split_configs=['1'] +rank0 [2026-11-15 02:37:06,762] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:37:06,766] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:37:06,813] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:37:07,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:37:08,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:37:08,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:37:08,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:37:09,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:37:09,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:37:10,142] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:37:10,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:37:10,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:37:11,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:37:11,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:37:12,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:37:12,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:37:13,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:37:13,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:37:13,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:37:13,911] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:37:13,919] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf +rank0 [2026-11-15 02:37:15,007] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_qq_pf +rank0 [2026-11-15 02:37:15,424] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:17,089] INFO : Plotted jets +rank0 [2026-11-15 02:37:18,119] INFO : Plotted jet ratio +rank0 [2026-11-15 02:37:18,129] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:37:19,647] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:37:20,102] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:21,427] INFO : Plotted jets +rank0 [2026-11-15 02:37:22,440] INFO : Plotted jet ratio +rank0 [2026-11-15 02:37:22,450] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:37:23,825] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_4000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:37:24,268] INFO : Plotted number of elements +rank0 [2026-11-15 02:37:25,569] INFO : Plotted jets +rank0 [2026-11-15 02:37:26,924] INFO : Plotted jet ratio +rank0 [2026-11-15 02:39:23,160] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 02:39:23,160] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:41:20,459] INFO : Step 5000/500000 rank0 | Train Loss: 4.7778 | Train Loss Unweighted: 1.1399 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2611 | Regression_cos_phi: 0.0277 | Regression_energy: 0.3852 | Regression_eta: 0.0867 | Regression_pt: 0.3604 | Regression_sin_phi: 0.0182 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1731 | Classification_binary: 0.2611 | Regression_cos_phi: 1.6256 | Regression_energy: 0.2305 | Regression_eta: 1.1261 | Regression_pt: 0.2155 | Regression_sin_phi: 1.1457 | LR: 2.13e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.2270s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0626 | max_loss: 0.0029 | max_backward: 0.1312 | max_optimizer_step: 0.0639 | max_tensorboard_logging: 0.7448 | max_tensorboard_flush: 0.0008 | Memory: rss_MB: 5607.3 | uss_MB: 4386.2 | child_rss_MB: 23785.4 | allocated_MB: 195.3 | reserved_MB: 22866.0 | inactive_split_MB: 268.7 +rank0 [2026-11-15 02:41:20,462] INFO : Running validation on rank0 +rank0 [2026-11-15 02:41:20,462] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:41:20,462] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:41:21,083] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 02:41:21,199] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:41:21,314] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:41:21,383] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:41:21,450] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:41:21,516] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:41:21,584] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:41:21,653] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:41:21,769] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:41:21,875] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:41:21,980] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:41:22,052] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:41:22,120] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:41:22,187] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:41:22,255] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:41:22,322] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:41:22,424] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:41:22,434] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:41:22,437] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:41:22,441] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:41:22,445] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:41:22,448] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:41:22,451] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:41:22,454] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:41:22,458] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:41:22,461] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:41:22,465] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:41:22,468] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:41:22,471] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:41:22,475] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:41:22,478] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:41:22,481] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:41:30,189] INFO : VALIDATION | Step=5000/500000 | Train Loss=4.7778 | Valid Loss=3.4041 | Train Loss Unweighted=1.1399 | Valid Loss Unweighted=1.1713 | Stale=0 | ETA=2233.8m +rank0 [2026-11-15 02:41:30,190] INFO : split_configs=['1'] +rank0 [2026-11-15 02:41:30,201] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:41:30,208] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:41:30,284] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:31,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:31,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:31,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:32,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:32,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:32,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:33,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:33,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:33,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:34,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:34,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:34,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:35,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:35,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:35,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:36,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:36,110] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:41:36,113] INFO : split_configs=['1'] +rank0 [2026-11-15 02:41:36,124] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:41:36,128] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:41:36,175] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:37,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:37,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:37,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:38,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:38,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:39,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:39,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:40,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:40,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:41,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:41,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:41,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:42,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:42,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:43,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:43,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:43,713] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:41:43,716] INFO : split_configs=['1'] +rank0 [2026-11-15 02:41:43,727] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:41:43,731] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:41:43,774] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:41:44,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:41:45,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:41:45,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:41:45,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:41:46,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:41:46,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:41:47,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:41:47,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:41:47,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:41:48,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:41:48,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:41:49,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:41:49,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:41:49,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:41:50,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:41:50,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:41:50,777] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:41:50,784] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf +rank0 [2026-11-15 02:41:51,867] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_qq_pf +rank0 [2026-11-15 02:41:52,290] INFO : Plotted number of elements +rank0 [2026-11-15 02:41:53,975] INFO : Plotted jets +rank0 [2026-11-15 02:41:54,982] INFO : Plotted jet ratio +rank0 [2026-11-15 02:41:54,993] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:41:56,523] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:41:56,964] INFO : Plotted number of elements +rank0 [2026-11-15 02:41:58,273] INFO : Plotted jets +rank0 [2026-11-15 02:41:59,295] INFO : Plotted jet ratio +rank0 [2026-11-15 02:41:59,304] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:42:00,701] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_5000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:42:01,140] INFO : Plotted number of elements +rank0 [2026-11-15 02:42:02,432] INFO : Plotted jets +rank0 [2026-11-15 02:42:03,409] INFO : Plotted jet ratio +rank0 [2026-11-15 02:45:57,687] INFO : Step 6000/500000 rank0 | Train Loss: 2.3339 | Train Loss Unweighted: 1.0330 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2483 | Regression_cos_phi: 0.0113 | Regression_energy: 0.3749 | Regression_eta: 0.0238 | Regression_pt: 0.3655 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1166 | Classification_binary: 0.2483 | Regression_cos_phi: 0.6613 | Regression_energy: 0.2244 | Regression_eta: 0.3093 | Regression_pt: 0.2186 | Regression_sin_phi: 0.5556 | LR: 2.19e-05 | DataLoad Time: 0.0023s | Model Forward Time: 1.8866s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0027 | max_forward: 0.0633 | max_loss: 0.0027 | max_backward: 0.1378 | max_optimizer_step: 0.0716 | max_tensorboard_logging: 1.3724 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5658.1 | uss_MB: 4437.1 | child_rss_MB: 23910.9 | allocated_MB: 199.4 | reserved_MB: 20068.0 | inactive_split_MB: 472.6 +rank0 [2026-11-15 02:45:57,691] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 02:45:57,759] INFO : Running validation on rank0 +rank0 [2026-11-15 02:45:57,759] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:45:57,759] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:45:58,381] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 02:45:58,503] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:45:58,618] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:45:58,690] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:45:58,759] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:45:58,829] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:45:58,898] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:45:58,968] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:45:59,069] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:45:59,172] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:45:59,276] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:45:59,343] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:45:59,411] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:45:59,478] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:45:59,545] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:45:59,613] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:45:59,715] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:45:59,723] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:45:59,725] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:45:59,726] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:45:59,727] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:45:59,728] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:45:59,729] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:45:59,730] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:45:59,732] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:45:59,733] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:45:59,734] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:45:59,750] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:45:59,751] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:45:59,752] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:45:59,753] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:45:59,754] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:46:07,574] INFO : VALIDATION | Step=6000/500000 | Train Loss=2.3339 | Valid Loss=3.3140 | Train Loss Unweighted=1.0330 | Valid Loss Unweighted=1.1478 | Stale=0 | ETA=2238.3m +rank0 [2026-11-15 02:46:07,576] INFO : split_configs=['1'] +rank0 [2026-11-15 02:46:07,586] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:46:07,589] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:46:07,658] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:46:08,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:46:08,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:46:09,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:46:09,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:46:09,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:46:10,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:46:10,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:46:10,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:46:11,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:46:11,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:46:11,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:46:12,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:46:12,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:46:12,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:46:13,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:46:13,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:46:13,583] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 02:46:13,586] INFO : split_configs=['1'] +rank0 [2026-11-15 02:46:13,597] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:46:13,601] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:46:13,640] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:46:14,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:46:15,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:46:15,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:46:15,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:46:16,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:46:16,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:46:17,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:46:17,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:46:18,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:46:18,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:46:19,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:46:19,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:46:19,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:46:20,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:46:20,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:46:21,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:46:21,307] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:46:21,311] INFO : split_configs=['1'] +rank0 [2026-11-15 02:46:21,322] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:46:21,325] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:46:21,365] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:46:22,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:46:22,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:46:23,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:46:23,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:46:23,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:46:24,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:46:24,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:46:25,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:46:25,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:46:25,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:46:26,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:46:26,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:46:27,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:46:27,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:46:27,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:46:28,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:46:28,348] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:46:28,358] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf +rank0 [2026-11-15 02:46:29,458] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_qq_pf +rank0 [2026-11-15 02:46:29,896] INFO : Plotted number of elements +rank0 [2026-11-15 02:46:31,564] INFO : Plotted jets +rank0 [2026-11-15 02:46:32,579] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:32,591] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:46:34,149] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:46:34,593] INFO : Plotted number of elements +rank0 [2026-11-15 02:46:35,911] INFO : Plotted jets +rank0 [2026-11-15 02:46:36,932] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:36,941] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:46:38,337] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_6000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:46:38,786] INFO : Plotted number of elements +rank0 [2026-11-15 02:46:40,093] INFO : Plotted jets +rank0 [2026-11-15 02:46:41,077] INFO : Plotted jet ratio +rank0 [2026-11-15 02:46:41,446] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 02:46:41,447] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:50:34,377] INFO : Step 7000/500000 rank0 | Train Loss: 3.5359 | Train Loss Unweighted: 1.2333 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.2855 | Regression_cos_phi: 0.0191 | Regression_energy: 0.4518 | Regression_eta: 0.0362 | Regression_pt: 0.4250 | Regression_sin_phi: 0.0152 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1795 | Classification_binary: 0.2855 | Regression_cos_phi: 1.1232 | Regression_energy: 0.2704 | Regression_eta: 0.4700 | Regression_pt: 0.2542 | Regression_sin_phi: 0.9532 | LR: 2.26e-05 | DataLoad Time: 0.0020s | Model Forward Time: 1.4969s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0593 | max_loss: 0.0029 | max_backward: 0.1280 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 1.0013 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5743.9 | uss_MB: 4522.9 | child_rss_MB: 23824.1 | allocated_MB: 196.8 | reserved_MB: 18104.0 | inactive_split_MB: 351.2 +rank0 [2026-11-15 02:50:34,381] INFO : Running validation on rank0 +rank0 [2026-11-15 02:50:34,381] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:50:34,381] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:50:34,997] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 02:50:35,118] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:50:35,234] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:50:35,304] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:50:35,373] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:50:35,440] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:50:35,507] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:50:35,574] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:50:35,675] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:50:35,779] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:50:35,880] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:50:35,949] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:50:36,016] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:50:36,083] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:50:36,150] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:50:36,217] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:50:36,321] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:50:36,343] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:50:36,356] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:50:36,360] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:50:36,381] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:50:36,384] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:50:36,385] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:50:36,386] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:50:36,388] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:50:36,388] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:50:36,389] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:50:36,391] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:50:36,392] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:50:36,402] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:50:36,403] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:50:36,404] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:50:44,096] INFO : VALIDATION | Step=7000/500000 | Train Loss=3.5359 | Valid Loss=3.2511 | Train Loss Unweighted=1.2333 | Valid Loss Unweighted=1.1241 | Stale=0 | ETA=2239.3m +rank0 [2026-11-15 02:50:44,097] INFO : split_configs=['1'] +rank0 [2026-11-15 02:50:44,134] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:50:44,139] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:50:44,216] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:50:44,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:50:45,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:50:45,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:50:46,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:50:46,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:50:46,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:50:47,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:50:47,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:50:47,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:50:48,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:50:48,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:50:48,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:50:49,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:50:49,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:50:49,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:50:49,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:50:50,052] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:50:50,055] INFO : split_configs=['1'] +rank0 [2026-11-15 02:50:50,101] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:50:50,104] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:50:50,148] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:50:51,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:50:51,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:50:51,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:50:52,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:50:52,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:50:53,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:50:53,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:50:54,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:50:54,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:50:55,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:50:55,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:50:55,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:50:56,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:50:56,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:50:57,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:50:57,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:50:57,769] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:50:57,772] INFO : split_configs=['1'] +rank0 [2026-11-15 02:50:57,806] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:50:57,811] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:50:57,857] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:50:58,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:50:59,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:50:59,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:50:59,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:51:00,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:51:00,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:51:01,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:51:01,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:51:02,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:51:02,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:51:02,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:51:03,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:51:03,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:51:04,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:51:04,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:51:04,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:51:04,940] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:51:04,951] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf +rank0 [2026-11-15 02:51:06,047] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_qq_pf +rank0 [2026-11-15 02:51:06,470] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:08,141] INFO : Plotted jets +rank0 [2026-11-15 02:51:09,158] INFO : Plotted jet ratio +rank0 [2026-11-15 02:51:09,170] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:51:10,716] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:51:11,159] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:12,468] INFO : Plotted jets +rank0 [2026-11-15 02:51:13,480] INFO : Plotted jet ratio +rank0 [2026-11-15 02:51:13,490] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:51:14,869] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_7000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:51:15,313] INFO : Plotted number of elements +rank0 [2026-11-15 02:51:16,609] INFO : Plotted jets +rank0 [2026-11-15 02:51:17,588] INFO : Plotted jet ratio +rank0 [2026-11-15 02:53:14,077] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 02:53:14,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 02:55:11,046] INFO : Step 8000/500000 rank0 | Train Loss: 3.4353 | Train Loss Unweighted: 1.2436 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2849 | Regression_cos_phi: 0.0204 | Regression_energy: 0.4547 | Regression_eta: 0.0368 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1502 | Classification_binary: 0.2849 | Regression_cos_phi: 1.1979 | Regression_energy: 0.2721 | Regression_eta: 0.4782 | Regression_pt: 0.2594 | Regression_sin_phi: 0.7925 | LR: 2.34e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.3671s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0584 | max_loss: 0.0029 | max_backward: 0.1237 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.8720 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5934.2 | uss_MB: 4713.2 | child_rss_MB: 23732.5 | allocated_MB: 199.5 | reserved_MB: 18154.0 | inactive_split_MB: 174.5 +rank0 [2026-11-15 02:55:11,048] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 02:55:11,116] INFO : Running validation on rank0 +rank0 [2026-11-15 02:55:11,116] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:55:11,116] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:55:11,729] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 02:55:11,846] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:55:11,963] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:55:12,035] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:55:12,103] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:55:12,171] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:55:12,240] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:55:12,309] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:55:12,414] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:55:12,518] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:55:12,620] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:55:12,689] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:55:12,757] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:55:12,823] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:55:12,891] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:55:12,958] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:55:13,061] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:55:13,071] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:55:13,075] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:55:13,078] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:55:13,082] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:55:13,085] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:55:13,089] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:55:13,092] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:55:13,095] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:55:13,099] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:55:13,102] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:55:13,105] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:55:13,108] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:55:13,111] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:55:13,115] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:55:13,119] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:55:20,819] INFO : VALIDATION | Step=8000/500000 | Train Loss=3.4353 | Valid Loss=3.2100 | Train Loss Unweighted=1.2436 | Valid Loss Unweighted=1.1045 | Stale=0 | ETA=2239.0m +rank0 [2026-11-15 02:55:20,821] INFO : split_configs=['1'] +rank0 [2026-11-15 02:55:20,831] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:55:20,838] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:55:20,917] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:21,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:22,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:22,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:22,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:23,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:23,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:23,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:24,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:24,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:24,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:25,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:25,429] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:25,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:26,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:26,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:26,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:26,766] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 02:55:26,769] INFO : split_configs=['1'] +rank0 [2026-11-15 02:55:26,779] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 02:55:26,783] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 02:55:26,828] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:27,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:28,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:28,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:29,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:29,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:29,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:30,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:30,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:31,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:31,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:32,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:32,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:33,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:33,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:33,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:34,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:34,430] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 02:55:34,434] INFO : split_configs=['1'] +rank0 [2026-11-15 02:55:34,468] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 02:55:34,471] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:55:34,518] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:55:35,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 02:55:35,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 02:55:36,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 02:55:36,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 02:55:37,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 02:55:37,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 02:55:37,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 02:55:38,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 02:55:38,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 02:55:39,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 02:55:39,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 02:55:39,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 02:55:40,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 02:55:40,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 02:55:41,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 02:55:41,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 02:55:41,623] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 02:55:41,630] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf +rank0 [2026-11-15 02:55:42,726] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_qq_pf +rank0 [2026-11-15 02:55:43,152] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:44,813] INFO : Plotted jets +rank0 [2026-11-15 02:55:45,843] INFO : Plotted jet ratio +rank0 [2026-11-15 02:55:45,854] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:55:47,365] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ttbar_pf +rank0 [2026-11-15 02:55:47,805] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:49,123] INFO : Plotted jets +rank0 [2026-11-15 02:55:50,137] INFO : Plotted jet ratio +rank0 [2026-11-15 02:55:50,147] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:55:51,538] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_8000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 02:55:51,979] INFO : Plotted number of elements +rank0 [2026-11-15 02:55:53,278] INFO : Plotted jets +rank0 [2026-11-15 02:55:54,260] INFO : Plotted jet ratio +rank0 [2026-11-15 02:59:47,747] INFO : Step 9000/500000 rank0 | Train Loss: 2.3927 | Train Loss Unweighted: 1.0289 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2665 | Regression_cos_phi: 0.0063 | Regression_energy: 0.3630 | Regression_eta: 0.0328 | Regression_pt: 0.3473 | Regression_sin_phi: 0.0127 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1103 | Classification_binary: 0.2665 | Regression_cos_phi: 0.3672 | Regression_energy: 0.2172 | Regression_eta: 0.4262 | Regression_pt: 0.2077 | Regression_sin_phi: 0.7976 | LR: 2.43e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.5016s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0699 | max_loss: 0.0027 | max_backward: 0.1544 | max_optimizer_step: 0.0643 | max_tensorboard_logging: 0.9426 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 5950.2 | uss_MB: 4729.3 | child_rss_MB: 23833.7 | allocated_MB: 198.3 | reserved_MB: 19218.0 | inactive_split_MB: 385.7 +rank0 [2026-11-15 02:59:47,751] INFO : Running validation on rank0 +rank0 [2026-11-15 02:59:47,751] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 02:59:47,752] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 02:59:48,382] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 02:59:48,506] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 02:59:48,619] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 02:59:48,691] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 02:59:48,759] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 02:59:48,826] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 02:59:48,894] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 02:59:48,962] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 02:59:49,067] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 02:59:49,173] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 02:59:49,277] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 02:59:49,345] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 02:59:49,414] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 02:59:49,483] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 02:59:49,550] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 02:59:49,617] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 02:59:49,721] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 02:59:49,741] INFO : plotted pt_target histogram +rank0 [2026-11-15 02:59:49,753] INFO : plotted pt_pred histogram +rank0 [2026-11-15 02:59:49,756] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 02:59:49,766] INFO : plotted eta_target histogram +rank0 [2026-11-15 02:59:49,779] INFO : plotted eta_pred histogram +rank0 [2026-11-15 02:59:49,782] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 02:59:49,792] INFO : plotted sphi_target histogram +rank0 [2026-11-15 02:59:49,805] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 02:59:49,808] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 02:59:49,819] INFO : plotted cphi_target histogram +rank0 [2026-11-15 02:59:49,822] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 02:59:49,825] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 02:59:49,827] INFO : plotted energy_target histogram +rank0 [2026-11-15 02:59:49,830] INFO : plotted energy_pred histogram +rank0 [2026-11-15 02:59:49,832] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 02:59:57,576] INFO : VALIDATION | Step=9000/500000 | Train Loss=2.3927 | Valid Loss=3.1234 | Train Loss Unweighted=1.0289 | Valid Loss Unweighted=1.0859 | Stale=0 | ETA=2237.9m +rank0 [2026-11-15 02:59:57,576] INFO : split_configs=['1'] +rank0 [2026-11-15 02:59:57,643] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 02:59:57,646] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 02:59:57,719] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 02:59:58,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 02:59:58,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 02:59:59,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 02:59:59,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 02:59:59,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:00:00,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:00:00,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:00:00,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:00:01,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:00:01,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:00:01,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:00:02,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:00:02,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:00:03,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:00:03,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:00:03,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:00:03,815] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:00:03,819] INFO : split_configs=['1'] +rank0 [2026-11-15 03:00:03,863] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:00:03,866] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:00:03,907] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:00:04,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:00:05,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:00:05,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:00:06,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:00:06,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:00:06,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:00:07,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:00:07,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:00:08,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:00:08,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:00:09,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:00:09,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:00:10,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:00:10,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:00:10,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:00:11,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:00:11,456] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:00:11,460] INFO : split_configs=['1'] +rank0 [2026-11-15 03:00:11,470] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:00:11,475] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:00:11,517] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:00:12,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:00:12,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:00:13,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:00:13,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:00:14,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:00:14,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:00:14,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:00:15,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:00:15,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:00:16,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:00:16,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:00:16,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:00:17,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:00:17,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:00:18,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:00:18,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:00:18,536] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:00:18,546] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf +rank0 [2026-11-15 03:00:19,657] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_qq_pf +rank0 [2026-11-15 03:00:20,088] INFO : Plotted number of elements +rank0 [2026-11-15 03:00:21,761] INFO : Plotted jets +rank0 [2026-11-15 03:00:22,777] INFO : Plotted jet ratio +rank0 [2026-11-15 03:00:22,789] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:00:24,356] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:00:24,792] INFO : Plotted number of elements +rank0 [2026-11-15 03:00:26,131] INFO : Plotted jets +rank0 [2026-11-15 03:00:27,152] INFO : Plotted jet ratio +rank0 [2026-11-15 03:00:27,160] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:00:28,582] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_9000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:00:29,024] INFO : Plotted number of elements +rank0 [2026-11-15 03:00:30,335] INFO : Plotted jets +rank0 [2026-11-15 03:00:31,319] INFO : Plotted jet ratio +rank0 [2026-11-15 03:00:31,712] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:00:31,712] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:04:26,047] INFO : Step 10000/500000 rank0 | Train Loss: 3.9013 | Train Loss Unweighted: 1.2087 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2662 | Regression_cos_phi: 0.0134 | Regression_energy: 0.4160 | Regression_eta: 0.1035 | Regression_pt: 0.3949 | Regression_sin_phi: 0.0141 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1303 | Classification_binary: 0.2662 | Regression_cos_phi: 0.7880 | Regression_energy: 0.2490 | Regression_eta: 1.3442 | Regression_pt: 0.2362 | Regression_sin_phi: 0.8874 | LR: 2.52e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.7200s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0651 | max_loss: 0.0028 | max_backward: 0.1420 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 1.1915 | max_tensorboard_flush: 0.0008 | Memory: rss_MB: 6110.1 | uss_MB: 4889.0 | child_rss_MB: 23751.4 | allocated_MB: 196.4 | reserved_MB: 18374.0 | inactive_split_MB: 289.6 +rank0 [2026-11-15 03:04:26,049] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 03:04:26,115] INFO : Running validation on rank0 +rank0 [2026-11-15 03:04:26,115] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:04:26,115] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:04:26,736] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 03:04:26,854] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:04:26,968] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:04:27,037] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:04:27,104] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:04:27,169] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:04:27,236] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:04:27,302] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:04:27,405] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:04:27,507] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:04:27,608] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:04:27,675] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:04:27,741] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:04:27,806] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:04:27,872] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:04:27,939] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:04:28,042] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:04:28,061] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:04:28,070] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:04:28,073] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:04:28,083] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:04:28,096] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:04:28,099] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:04:28,102] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:04:28,105] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:04:28,107] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:04:28,110] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:04:28,112] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:04:28,115] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:04:28,117] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:04:28,120] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:04:28,122] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:04:35,856] INFO : VALIDATION | Step=10000/500000 | Train Loss=3.9013 | Valid Loss=3.1129 | Train Loss Unweighted=1.2087 | Valid Loss Unweighted=1.0716 | Stale=0 | ETA=2237.2m +rank0 [2026-11-15 03:04:35,857] INFO : split_configs=['1'] +rank0 [2026-11-15 03:04:35,906] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:04:35,911] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:04:35,992] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:04:36,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:04:37,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:04:37,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:04:37,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:04:38,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:04:38,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:04:38,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:04:39,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:04:39,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:04:39,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:04:40,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:04:40,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:04:40,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:04:41,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:04:41,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:04:41,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:04:41,940] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:04:41,943] INFO : split_configs=['1'] +rank0 [2026-11-15 03:04:41,985] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:04:41,988] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:04:42,035] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:04:42,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:04:43,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:04:43,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:04:44,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:04:44,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:04:45,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:04:45,554] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:04:46,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:04:46,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:04:46,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:04:47,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:04:47,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:04:48,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:04:48,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:04:49,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:04:49,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:04:49,583] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:04:49,586] INFO : split_configs=['1'] +rank0 [2026-11-15 03:04:49,623] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:04:49,629] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:04:49,675] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:04:50,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:04:50,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:04:51,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:04:51,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:04:52,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:04:52,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:04:53,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:04:53,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:04:53,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:04:54,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:04:54,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:04:55,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:04:55,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:04:55,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:04:56,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:04:56,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:04:56,770] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:04:56,777] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf +rank0 [2026-11-15 03:04:57,868] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_qq_pf +rank0 [2026-11-15 03:04:58,303] INFO : Plotted number of elements +rank0 [2026-11-15 03:04:59,977] INFO : Plotted jets +rank0 [2026-11-15 03:05:00,997] INFO : Plotted jet ratio +rank0 [2026-11-15 03:05:01,007] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:05:02,566] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:05:02,997] INFO : Plotted number of elements +rank0 [2026-11-15 03:05:04,325] INFO : Plotted jets +rank0 [2026-11-15 03:05:05,339] INFO : Plotted jet ratio +rank0 [2026-11-15 03:05:05,348] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:05:06,760] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_10000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:05:07,196] INFO : Plotted number of elements +rank0 [2026-11-15 03:05:08,499] INFO : Plotted jets +rank0 [2026-11-15 03:05:09,470] INFO : Plotted jet ratio +rank0 [2026-11-15 03:07:05,250] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 03:07:05,251] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:09:02,394] INFO : Step 11000/500000 rank0 | Train Loss: 2.6454 | Train Loss Unweighted: 0.9620 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2613 | Regression_cos_phi: 0.0130 | Regression_energy: 0.3284 | Regression_eta: 0.0398 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1394 | Classification_binary: 0.2613 | Regression_cos_phi: 0.7652 | Regression_energy: 0.1966 | Regression_eta: 0.5175 | Regression_pt: 0.1852 | Regression_sin_phi: 0.5803 | LR: 2.63e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.3790s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0651 | max_loss: 0.0028 | max_backward: 0.1412 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.8517 | max_tensorboard_flush: 0.0009 | Memory: rss_MB: 6158.4 | uss_MB: 4937.4 | child_rss_MB: 23908.5 | allocated_MB: 198.3 | reserved_MB: 17166.0 | inactive_split_MB: 369.7 +rank0 [2026-11-15 03:09:02,396] INFO : Running validation on rank0 +rank0 [2026-11-15 03:09:02,396] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:09:02,396] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:09:03,009] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 03:09:03,129] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:09:03,243] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:09:03,313] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:09:03,380] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:09:03,446] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:09:03,514] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:09:03,581] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:09:03,686] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:09:03,787] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:09:03,887] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:09:03,953] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:09:04,018] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:09:04,083] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:09:04,149] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:09:04,214] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:09:04,314] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:09:04,321] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:09:04,324] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:09:04,326] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:09:04,329] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:09:04,331] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:09:04,334] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:09:04,336] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:09:04,339] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:09:04,341] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:09:04,343] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:09:04,346] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:09:04,348] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:09:04,351] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:09:04,353] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:09:04,356] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:09:12,043] INFO : VALIDATION | Step=11000/500000 | Train Loss=2.6454 | Valid Loss=3.0495 | Train Loss Unweighted=0.9620 | Valid Loss Unweighted=1.0586 | Stale=0 | ETA=2234.3m +rank0 [2026-11-15 03:09:12,044] INFO : split_configs=['1'] +rank0 [2026-11-15 03:09:12,054] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:09:12,075] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:09:12,147] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:09:12,916] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:09:13,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:09:13,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:09:13,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:09:14,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:09:14,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:09:14,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:09:15,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:09:15,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:09:15,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:09:16,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:09:16,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:09:16,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:09:17,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:09:17,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:09:17,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:09:18,014] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:09:18,017] INFO : split_configs=['1'] +rank0 [2026-11-15 03:09:18,028] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:09:18,032] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:09:18,079] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:09:19,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:09:19,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:09:19,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:09:20,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:09:20,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:09:21,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:09:21,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:09:22,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:09:22,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:09:22,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:09:23,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:09:23,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:09:24,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:09:24,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:09:25,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:09:25,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:09:25,688] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:09:25,691] INFO : split_configs=['1'] +rank0 [2026-11-15 03:09:25,702] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:09:25,706] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:09:25,756] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:09:26,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:09:27,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:09:27,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:09:27,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:09:28,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:09:28,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:09:29,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:09:29,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:09:29,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:09:30,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:09:30,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:09:31,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:09:31,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:09:31,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:09:32,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:09:32,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:09:32,843] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:09:32,851] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf +rank0 [2026-11-15 03:09:33,947] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_qq_pf +rank0 [2026-11-15 03:09:34,369] INFO : Plotted number of elements +rank0 [2026-11-15 03:09:36,072] INFO : Plotted jets +rank0 [2026-11-15 03:09:37,090] INFO : Plotted jet ratio +rank0 [2026-11-15 03:09:37,100] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:09:38,710] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:09:39,145] INFO : Plotted number of elements +rank0 [2026-11-15 03:09:40,463] INFO : Plotted jets +rank0 [2026-11-15 03:09:41,479] INFO : Plotted jet ratio +rank0 [2026-11-15 03:09:41,487] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:09:42,923] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_11000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:09:43,367] INFO : Plotted number of elements +rank0 [2026-11-15 03:09:44,666] INFO : Plotted jets +rank0 [2026-11-15 03:09:45,644] INFO : Plotted jet ratio +rank0 [2026-11-15 03:13:39,004] INFO : Step 12000/500000 rank0 | Train Loss: 2.7292 | Train Loss Unweighted: 1.0822 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2527 | Regression_cos_phi: 0.0121 | Regression_energy: 0.3935 | Regression_eta: 0.0387 | Regression_pt: 0.3739 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1186 | Classification_binary: 0.2527 | Regression_cos_phi: 0.7119 | Regression_energy: 0.2355 | Regression_eta: 0.5022 | Regression_pt: 0.2236 | Regression_sin_phi: 0.6847 | LR: 2.75e-05 | DataLoad Time: 0.0020s | Model Forward Time: 1.3928s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0659 | max_loss: 0.0027 | max_backward: 0.1404 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0010 | Memory: rss_MB: 4566.1 | uss_MB: 3345.1 | child_rss_MB: 23727.5 | allocated_MB: 196.8 | reserved_MB: 21524.0 | inactive_split_MB: 155.2 +rank0 [2026-11-15 03:13:39,005] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 03:13:39,073] INFO : Running validation on rank0 +rank0 [2026-11-15 03:13:39,074] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:13:39,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:13:39,698] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 03:13:39,823] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:13:39,935] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:13:40,005] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:13:40,075] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:13:40,143] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:13:40,210] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:13:40,278] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:13:40,384] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:13:40,486] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:13:40,588] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:13:40,654] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:13:40,719] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:13:40,784] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:13:40,849] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:13:40,915] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:13:41,016] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:13:41,023] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:13:41,026] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:13:41,028] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:13:41,031] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:13:41,034] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:13:41,036] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:13:41,039] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:13:41,041] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:13:41,044] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:13:41,046] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:13:41,049] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:13:41,051] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:13:41,054] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:13:41,056] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:13:41,059] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:13:48,797] INFO : VALIDATION | Step=12000/500000 | Train Loss=2.7292 | Valid Loss=3.0405 | Train Loss Unweighted=1.0822 | Valid Loss Unweighted=1.0477 | Stale=0 | ETA=2231.5m +rank0 [2026-11-15 03:13:48,797] INFO : split_configs=['1'] +rank0 [2026-11-15 03:13:48,808] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:13:48,811] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:13:48,886] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:13:49,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:13:49,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:13:50,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:13:50,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:13:51,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:13:51,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:13:51,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:13:52,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:13:52,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:13:52,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:13:53,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:13:53,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:13:53,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:13:54,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:13:54,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:13:54,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:13:54,743] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:13:54,746] INFO : split_configs=['1'] +rank0 [2026-11-15 03:13:54,757] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:13:54,760] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:13:54,802] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:13:55,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:13:56,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:13:56,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:13:57,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:13:57,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:13:57,890] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:13:58,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:13:58,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:13:59,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:13:59,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:14:00,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:14:00,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:14:01,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:14:01,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:14:01,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:14:02,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:14:02,374] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:14:02,380] INFO : split_configs=['1'] +rank0 [2026-11-15 03:14:02,392] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:14:02,395] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:14:02,438] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:14:03,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:14:03,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:14:04,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:14:04,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:14:04,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:14:05,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:14:05,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:14:06,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:14:06,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:14:07,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:14:07,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:14:07,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:14:08,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:14:08,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:14:09,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:14:09,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:14:09,550] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:14:09,557] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf +rank0 [2026-11-15 03:14:10,697] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_qq_pf +rank0 [2026-11-15 03:14:11,137] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:12,835] INFO : Plotted jets +rank0 [2026-11-15 03:14:13,860] INFO : Plotted jet ratio +rank0 [2026-11-15 03:14:13,872] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:14:15,473] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:14:15,933] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:17,270] INFO : Plotted jets +rank0 [2026-11-15 03:14:18,292] INFO : Plotted jet ratio +rank0 [2026-11-15 03:14:18,300] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:14:19,717] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_12000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:14:20,175] INFO : Plotted number of elements +rank0 [2026-11-15 03:14:21,481] INFO : Plotted jets +rank0 [2026-11-15 03:14:22,463] INFO : Plotted jet ratio +rank0 [2026-11-15 03:14:22,846] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 03:14:22,847] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:18:16,444] INFO : Step 13000/500000 rank0 | Train Loss: 3.1323 | Train Loss Unweighted: 1.2570 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2596 | Regression_cos_phi: 0.0180 | Regression_energy: 0.4838 | Regression_eta: 0.0260 | Regression_pt: 0.4567 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1397 | Classification_binary: 0.2596 | Regression_cos_phi: 1.0580 | Regression_energy: 0.2896 | Regression_eta: 0.3372 | Regression_pt: 0.2732 | Regression_sin_phi: 0.7750 | LR: 2.88e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.4011s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0652 | max_loss: 0.0030 | max_backward: 0.1419 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.8528 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 6324.3 | uss_MB: 5103.2 | child_rss_MB: 23860.7 | allocated_MB: 198.1 | reserved_MB: 18360.0 | inactive_split_MB: 177.9 +rank0 [2026-11-15 03:18:16,448] INFO : Running validation on rank0 +rank0 [2026-11-15 03:18:16,448] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:18:16,448] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:18:17,071] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 03:18:17,191] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:18:17,304] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:18:17,372] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:18:17,438] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:18:17,502] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:18:17,567] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:18:17,633] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:18:17,739] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:18:17,845] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:18:17,950] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:18:18,018] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:18:18,087] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:18:18,154] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:18:18,220] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:18:18,285] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:18:18,386] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:18:18,393] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:18:18,396] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:18:18,399] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:18:18,402] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:18:18,404] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:18:18,407] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:18:18,409] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:18:18,412] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:18:18,414] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:18:18,416] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:18:18,419] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:18:18,421] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:18:18,424] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:18:18,426] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:18:18,429] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:18:26,146] INFO : VALIDATION | Step=13000/500000 | Train Loss=3.1323 | Valid Loss=3.0005 | Train Loss Unweighted=1.2570 | Valid Loss Unweighted=1.0367 | Stale=0 | ETA=2228.8m +rank0 [2026-11-15 03:18:26,146] INFO : split_configs=['1'] +rank0 [2026-11-15 03:18:26,158] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:18:26,161] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:18:26,241] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:27,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:27,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:27,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:28,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:28,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:28,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:29,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:29,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:29,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:30,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:30,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:30,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:31,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:31,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:31,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:32,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:32,241] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:18:32,244] INFO : split_configs=['1'] +rank0 [2026-11-15 03:18:32,255] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:18:32,263] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:18:32,313] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:33,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:33,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:34,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:34,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:35,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:35,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:35,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:36,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:36,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:37,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:37,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:38,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:38,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:39,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:39,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:39,916] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:39,987] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:18:39,991] INFO : split_configs=['1'] +rank0 [2026-11-15 03:18:40,002] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:18:40,005] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:18:40,057] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:18:40,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:18:41,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:18:41,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:18:42,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:18:42,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:18:43,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:18:43,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:18:43,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:18:44,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:18:44,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:18:45,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:18:45,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:18:45,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:18:46,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:18:46,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:18:47,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:18:47,219] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:18:47,226] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf +rank0 [2026-11-15 03:18:48,341] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_qq_pf +rank0 [2026-11-15 03:18:48,768] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:50,484] INFO : Plotted jets +rank0 [2026-11-15 03:18:51,516] INFO : Plotted jet ratio +rank0 [2026-11-15 03:18:51,529] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:18:53,089] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:18:53,527] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:54,855] INFO : Plotted jets +rank0 [2026-11-15 03:18:55,883] INFO : Plotted jet ratio +rank0 [2026-11-15 03:18:55,893] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:18:57,285] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_13000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:18:57,736] INFO : Plotted number of elements +rank0 [2026-11-15 03:18:59,052] INFO : Plotted jets +rank0 [2026-11-15 03:19:00,045] INFO : Plotted jet ratio +rank0 [2026-11-15 03:20:56,366] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 03:20:56,366] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:22:53,704] INFO : Step 14000/500000 rank0 | Train Loss: 2.9143 | Train Loss Unweighted: 1.0571 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.2785 | Regression_cos_phi: 0.0120 | Regression_energy: 0.3733 | Regression_eta: 0.0279 | Regression_pt: 0.3493 | Regression_sin_phi: 0.0155 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1575 | Classification_binary: 0.2785 | Regression_cos_phi: 0.7069 | Regression_energy: 0.2234 | Regression_eta: 0.3623 | Regression_pt: 0.2089 | Regression_sin_phi: 0.9768 | LR: 3.02e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.3310s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0591 | max_loss: 0.0030 | max_backward: 0.1265 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0009 | Memory: rss_MB: 6405.4 | uss_MB: 5196.3 | child_rss_MB: 23666.8 | allocated_MB: 196.6 | reserved_MB: 18680.0 | inactive_split_MB: 177.4 +rank0 [2026-11-15 03:22:53,709] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 03:22:53,774] INFO : Running validation on rank0 +rank0 [2026-11-15 03:22:53,774] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:22:53,774] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:22:54,391] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 03:22:54,508] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:22:54,618] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:22:54,686] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:22:54,753] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:22:54,818] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:22:54,885] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:22:54,952] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:22:55,059] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:22:55,165] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:22:55,268] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:22:55,333] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:22:55,398] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:22:55,462] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:22:55,528] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:22:55,592] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:22:55,692] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:22:55,699] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:22:55,702] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:22:55,704] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:22:55,707] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:22:55,710] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:22:55,712] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:22:55,714] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:22:55,717] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:22:55,719] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:22:55,721] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:22:55,724] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:22:55,726] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:22:55,729] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:22:55,731] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:22:55,734] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:23:03,435] INFO : VALIDATION | Step=14000/500000 | Train Loss=2.9143 | Valid Loss=2.9835 | Train Loss Unweighted=1.0571 | Valid Loss Unweighted=1.0255 | Stale=0 | ETA=2225.8m +rank0 [2026-11-15 03:23:03,435] INFO : split_configs=['1'] +rank0 [2026-11-15 03:23:03,446] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:23:03,454] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:23:03,546] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:23:04,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:23:04,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:23:05,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:23:05,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:23:05,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:23:06,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:23:06,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:23:06,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:23:07,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:23:07,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:23:07,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:23:08,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:23:08,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:23:08,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:23:09,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:23:09,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:23:09,441] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:23:09,443] INFO : split_configs=['1'] +rank0 [2026-11-15 03:23:09,455] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:23:09,458] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:23:09,511] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:23:10,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:23:10,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:23:11,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:23:11,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:23:12,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:23:12,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:23:13,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:23:13,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:23:14,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:23:14,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:23:14,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:23:15,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:23:15,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:23:16,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:23:16,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:23:17,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:23:17,156] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:23:17,160] INFO : split_configs=['1'] +rank0 [2026-11-15 03:23:17,171] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:23:17,186] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:23:17,236] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:23:18,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:23:18,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:23:18,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:23:19,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:23:19,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:23:20,217] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:23:20,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:23:21,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:23:21,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:23:21,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:23:22,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:23:22,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:23:23,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:23:23,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:23:23,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:23:24,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:23:24,329] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:23:24,337] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf +rank0 [2026-11-15 03:23:25,442] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_qq_pf +rank0 [2026-11-15 03:23:25,867] INFO : Plotted number of elements +rank0 [2026-11-15 03:23:27,548] INFO : Plotted jets +rank0 [2026-11-15 03:23:28,575] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:28,586] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:23:30,140] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:23:30,584] INFO : Plotted number of elements +rank0 [2026-11-15 03:23:31,899] INFO : Plotted jets +rank0 [2026-11-15 03:23:32,919] INFO : Plotted jet ratio +rank0 [2026-11-15 03:23:32,928] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:23:34,315] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_14000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:23:34,757] INFO : Plotted number of elements +rank0 [2026-11-15 03:23:36,065] INFO : Plotted jets +rank0 [2026-11-15 03:23:37,046] INFO : Plotted jet ratio +rank0 [2026-11-15 03:27:29,902] INFO : Step 15000/500000 rank0 | Train Loss: 2.4760 | Train Loss Unweighted: 0.9661 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2485 | Regression_cos_phi: 0.0127 | Regression_energy: 0.3365 | Regression_eta: 0.0343 | Regression_pt: 0.3251 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0850 | Classification_binary: 0.2485 | Regression_cos_phi: 0.7448 | Regression_energy: 0.2014 | Regression_eta: 0.4450 | Regression_pt: 0.1944 | Regression_sin_phi: 0.5569 | LR: 3.17e-05 | DataLoad Time: 0.0024s | Model Forward Time: 1.5519s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0024 | max_forward: 0.0557 | max_loss: 0.0028 | max_backward: 0.1190 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 1.0551 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 4875.8 | uss_MB: 3666.6 | child_rss_MB: 23529.2 | allocated_MB: 194.5 | reserved_MB: 17698.0 | inactive_split_MB: 171.5 +rank0 [2026-11-15 03:27:29,905] INFO : Running validation on rank0 +rank0 [2026-11-15 03:27:29,905] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:27:29,905] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:27:30,528] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 03:27:30,651] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:27:30,763] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:27:30,832] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:27:30,897] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:27:30,962] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:27:31,029] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:27:31,099] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:27:31,206] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:27:31,312] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:27:31,415] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:27:31,482] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:27:31,548] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:27:31,612] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:27:31,677] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:27:31,743] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:27:31,843] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:27:31,850] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:27:31,853] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:27:31,855] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:27:31,858] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:27:31,860] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:27:31,863] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:27:31,865] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:27:31,868] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:27:31,870] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:27:31,872] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:27:31,875] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:27:31,877] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:27:31,880] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:27:31,882] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:27:31,885] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:27:39,602] INFO : VALIDATION | Step=15000/500000 | Train Loss=2.4760 | Valid Loss=2.9111 | Train Loss Unweighted=0.9661 | Valid Loss Unweighted=1.0166 | Stale=0 | ETA=2222.0m +rank0 [2026-11-15 03:27:39,603] INFO : split_configs=['1'] +rank0 [2026-11-15 03:27:39,647] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:27:39,650] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:27:39,723] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:27:40,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:27:40,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:27:41,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:27:41,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:27:41,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:27:42,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:27:42,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:27:42,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:27:43,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:27:43,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:27:43,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:27:44,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:27:44,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:27:44,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:27:45,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:27:45,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:27:45,576] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:27:45,580] INFO : split_configs=['1'] +rank0 [2026-11-15 03:27:45,639] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:27:45,642] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:27:45,689] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:27:46,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:27:47,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:27:47,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:27:47,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:27:48,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:27:48,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:27:49,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:27:49,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:27:50,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:27:50,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:27:51,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:27:51,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:27:51,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:27:52,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:27:52,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:27:53,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:27:53,335] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:27:53,338] INFO : split_configs=['1'] +rank0 [2026-11-15 03:27:53,378] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:27:53,381] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:27:53,429] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:27:54,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:27:54,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:27:55,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:27:55,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:27:55,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:27:56,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:27:56,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:27:57,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:27:57,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:27:57,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:27:58,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:27:58,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:27:59,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:27:59,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:28:00,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:28:00,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:28:00,505] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:28:00,523] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf +rank0 [2026-11-15 03:28:01,648] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_qq_pf +rank0 [2026-11-15 03:28:02,088] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:03,800] INFO : Plotted jets +rank0 [2026-11-15 03:28:04,867] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:04,879] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:28:06,478] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:28:06,921] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:08,288] INFO : Plotted jets +rank0 [2026-11-15 03:28:09,307] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:09,316] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:28:10,774] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_15000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:28:11,216] INFO : Plotted number of elements +rank0 [2026-11-15 03:28:12,536] INFO : Plotted jets +rank0 [2026-11-15 03:28:13,521] INFO : Plotted jet ratio +rank0 [2026-11-15 03:28:13,919] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 03:28:13,919] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:32:08,677] INFO : Step 16000/500000 rank0 | Train Loss: 2.4377 | Train Loss Unweighted: 1.0890 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2703 | Regression_cos_phi: 0.0096 | Regression_energy: 0.3958 | Regression_eta: 0.0417 | Regression_pt: 0.3632 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1073 | Classification_binary: 0.2703 | Regression_cos_phi: 0.5633 | Regression_energy: 0.2368 | Regression_eta: 0.5419 | Regression_pt: 0.2172 | Regression_sin_phi: 0.5009 | LR: 3.33e-05 | DataLoad Time: 0.0022s | Model Forward Time: 2.7058s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0617 | max_loss: 0.0030 | max_backward: 0.1307 | max_optimizer_step: 0.0703 | max_tensorboard_logging: 2.1717 | max_tensorboard_flush: 0.0012 | Memory: rss_MB: 6363.9 | uss_MB: 5218.2 | child_rss_MB: 23765.8 | allocated_MB: 199.0 | reserved_MB: 20632.0 | inactive_split_MB: 167.0 +rank0 [2026-11-15 03:32:08,680] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 03:32:08,744] INFO : Running validation on rank0 +rank0 [2026-11-15 03:32:08,744] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:32:08,744] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:32:09,368] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 03:32:09,487] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:32:09,603] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:32:09,673] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:32:09,740] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:32:09,806] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:32:09,872] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:32:09,938] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:32:10,050] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:32:10,155] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:32:10,257] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:32:10,325] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:32:10,391] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:32:10,456] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:32:10,521] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:32:10,586] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:32:10,688] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:32:10,710] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:32:10,722] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:32:10,725] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:32:10,728] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:32:10,731] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:32:10,733] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:32:10,736] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:32:10,752] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:32:10,756] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:32:10,771] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:32:10,773] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:32:10,776] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:32:10,778] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:32:10,781] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:32:10,783] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:32:18,510] INFO : VALIDATION | Step=16000/500000 | Train Loss=2.4377 | Valid Loss=2.9038 | Train Loss Unweighted=1.0890 | Valid Loss Unweighted=1.0098 | Stale=0 | ETA=2219.4m +rank0 [2026-11-15 03:32:18,510] INFO : split_configs=['1'] +rank0 [2026-11-15 03:32:18,560] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:32:18,568] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:32:18,650] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:19,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:19,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:20,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:20,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:20,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:21,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:21,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:21,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:22,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:22,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:22,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:23,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:23,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:23,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:24,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:24,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:24,620] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:32:24,623] INFO : split_configs=['1'] +rank0 [2026-11-15 03:32:24,712] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:32:24,728] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:32:24,779] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:25,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:26,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:26,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:27,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:27,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:27,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:28,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:28,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:29,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:29,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:30,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:30,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:31,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:31,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:31,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:32,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:32,449] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:32:32,453] INFO : split_configs=['1'] +rank0 [2026-11-15 03:32:32,557] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:32:32,561] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:32:32,616] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:32:33,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:32:33,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:32:34,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:32:34,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:32:35,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:32:35,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:32:36,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:32:36,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:32:36,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:32:37,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:32:37,656] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:32:38,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:32:38,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:32:38,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:32:39,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:32:39,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:32:39,818] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:32:39,837] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf +rank0 [2026-11-15 03:32:40,947] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_qq_pf +rank0 [2026-11-15 03:32:41,384] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:43,079] INFO : Plotted jets +rank0 [2026-11-15 03:32:44,108] INFO : Plotted jet ratio +rank0 [2026-11-15 03:32:44,122] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:32:45,709] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:32:46,165] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:47,494] INFO : Plotted jets +rank0 [2026-11-15 03:32:48,523] INFO : Plotted jet ratio +rank0 [2026-11-15 03:32:48,533] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:32:49,955] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_16000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:32:50,407] INFO : Plotted number of elements +rank0 [2026-11-15 03:32:51,722] INFO : Plotted jets +rank0 [2026-11-15 03:32:52,714] INFO : Plotted jet ratio +rank0 [2026-11-15 03:34:48,766] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 03:34:48,766] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:36:46,317] INFO : Step 17000/500000 rank0 | Train Loss: 2.0359 | Train Loss Unweighted: 0.8891 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2597 | Regression_cos_phi: 0.0061 | Regression_energy: 0.2984 | Regression_eta: 0.0271 | Regression_pt: 0.2874 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0868 | Classification_binary: 0.2597 | Regression_cos_phi: 0.3558 | Regression_energy: 0.1786 | Regression_eta: 0.3519 | Regression_pt: 0.1719 | Regression_sin_phi: 0.6312 | LR: 3.51e-05 | DataLoad Time: 0.0019s | Model Forward Time: 1.5780s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0663 | max_loss: 0.0029 | max_backward: 0.1438 | max_optimizer_step: 0.0712 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6632.7 | uss_MB: 5423.6 | child_rss_MB: 23737.1 | allocated_MB: 196.0 | reserved_MB: 19958.0 | inactive_split_MB: 146.0 +rank0 [2026-11-15 03:36:46,320] INFO : Running validation on rank0 +rank0 [2026-11-15 03:36:46,320] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:36:46,320] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:36:46,935] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 03:36:47,054] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:36:47,168] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:36:47,235] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:36:47,300] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:36:47,365] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:36:47,431] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:36:47,497] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:36:47,602] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:36:47,706] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:36:47,810] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:36:47,878] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:36:47,945] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:36:48,011] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:36:48,079] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:36:48,144] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:36:48,244] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:36:48,250] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:36:48,251] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:36:48,252] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:36:48,253] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:36:48,255] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:36:48,256] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:36:48,257] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:36:48,258] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:36:48,259] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:36:48,277] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:36:48,278] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:36:48,279] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:36:48,280] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:36:48,281] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:36:48,282] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:36:56,030] INFO : VALIDATION | Step=17000/500000 | Train Loss=2.0359 | Valid Loss=2.8832 | Train Loss Unweighted=0.8891 | Valid Loss Unweighted=0.9973 | Stale=0 | ETA=2215.9m +rank0 [2026-11-15 03:36:56,030] INFO : split_configs=['1'] +rank0 [2026-11-15 03:36:56,098] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:36:56,103] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:36:56,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:36:56,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:36:57,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:36:57,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:36:58,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:36:58,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:36:58,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:36:59,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:36:59,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:36:59,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:37:00,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:00,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:00,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:01,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:01,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:01,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:02,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:02,101] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:37:02,105] INFO : split_configs=['1'] +rank0 [2026-11-15 03:37:02,167] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:37:02,171] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:37:02,220] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:37:03,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:37:03,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:37:04,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:37:04,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:37:04,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:37:05,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:37:05,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:37:06,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:37:06,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:37:07,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:07,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:08,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:08,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:08,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:09,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:09,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:09,868] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:37:09,872] INFO : split_configs=['1'] +rank0 [2026-11-15 03:37:09,934] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:37:09,937] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:37:09,985] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:37:10,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:37:11,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:37:11,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:37:12,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:37:12,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:37:12,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:37:13,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:37:13,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:37:14,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:37:14,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:37:14,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:37:15,382] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:37:15,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:37:16,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:37:16,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:37:17,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:37:17,103] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:37:17,114] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf +rank0 [2026-11-15 03:37:18,241] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_qq_pf +rank0 [2026-11-15 03:37:18,670] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:20,387] INFO : Plotted jets +rank0 [2026-11-15 03:37:21,407] INFO : Plotted jet ratio +rank0 [2026-11-15 03:37:21,424] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:37:23,042] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:37:23,479] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:24,811] INFO : Plotted jets +rank0 [2026-11-15 03:37:25,855] INFO : Plotted jet ratio +rank0 [2026-11-15 03:37:25,865] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:37:27,296] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_17000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:37:27,741] INFO : Plotted number of elements +rank0 [2026-11-15 03:37:29,045] INFO : Plotted jets +rank0 [2026-11-15 03:37:30,028] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:22,927] INFO : Step 18000/500000 rank0 | Train Loss: 2.3095 | Train Loss Unweighted: 1.0294 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2439 | Regression_cos_phi: 0.0072 | Regression_energy: 0.3711 | Regression_eta: 0.0273 | Regression_pt: 0.3678 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1109 | Classification_binary: 0.2439 | Regression_cos_phi: 0.4242 | Regression_energy: 0.2221 | Regression_eta: 0.3551 | Regression_pt: 0.2200 | Regression_sin_phi: 0.7333 | LR: 3.69e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.3672s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0600 | max_loss: 0.0027 | max_backward: 0.1286 | max_optimizer_step: 0.0700 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6604.0 | uss_MB: 5394.9 | child_rss_MB: 23820.7 | allocated_MB: 198.4 | reserved_MB: 18404.0 | inactive_split_MB: 183.6 +rank0 [2026-11-15 03:41:22,929] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 03:41:22,998] INFO : Running validation on rank0 +rank0 [2026-11-15 03:41:22,998] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:41:22,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:41:23,614] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 03:41:23,731] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:41:23,843] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:41:23,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:41:23,978] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:41:24,043] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:41:24,112] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:41:24,180] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:41:24,288] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:41:24,393] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:41:24,495] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:41:24,560] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:41:24,626] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:41:24,691] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:41:24,756] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:41:24,821] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:41:24,921] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:41:24,928] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:41:24,931] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:41:24,933] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:41:24,935] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:41:24,938] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:41:24,940] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:41:24,943] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:41:24,945] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:41:24,948] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:41:24,950] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:41:24,953] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:41:24,955] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:41:24,957] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:41:24,960] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:41:24,962] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:41:32,600] INFO : VALIDATION | Step=18000/500000 | Train Loss=2.3095 | Valid Loss=2.9102 | Train Loss Unweighted=1.0294 | Valid Loss Unweighted=0.9902 | Stale=1 | ETA=2211.9m +rank0 [2026-11-15 03:41:32,601] INFO : split_configs=['1'] +rank0 [2026-11-15 03:41:32,612] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:41:32,616] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:41:32,703] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:33,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:33,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:34,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:34,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:34,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:35,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:35,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:35,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:36,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:36,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:36,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:37,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:37,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:37,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:38,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:38,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:38,658] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:41:38,661] INFO : split_configs=['1'] +rank0 [2026-11-15 03:41:38,698] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:41:38,701] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:41:38,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:39,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:40,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:40,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:41,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:41,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:41,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:42,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:42,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:43,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:43,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:44,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:44,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:45,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:45,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:45,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:46,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:46,483] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:41:46,486] INFO : split_configs=['1'] +rank0 [2026-11-15 03:41:46,528] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:41:46,535] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:41:46,590] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:41:47,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:41:47,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:41:48,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:41:48,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:41:49,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:41:49,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:41:50,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:41:50,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:41:50,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:41:51,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:41:51,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:41:52,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:41:52,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:41:52,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:41:53,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:41:53,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:41:53,749] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:41:53,756] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf +rank0 [2026-11-15 03:41:54,878] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_qq_pf +rank0 [2026-11-15 03:41:55,304] INFO : Plotted number of elements +rank0 [2026-11-15 03:41:57,012] INFO : Plotted jets +rank0 [2026-11-15 03:41:58,064] INFO : Plotted jet ratio +rank0 [2026-11-15 03:41:58,078] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:41:59,645] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:42:00,076] INFO : Plotted number of elements +rank0 [2026-11-15 03:42:01,412] INFO : Plotted jets +rank0 [2026-11-15 03:42:02,431] INFO : Plotted jet ratio +rank0 [2026-11-15 03:42:02,440] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:42:03,881] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_18000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:42:04,313] INFO : Plotted number of elements +rank0 [2026-11-15 03:42:05,621] INFO : Plotted jets +rank0 [2026-11-15 03:42:06,597] INFO : Plotted jet ratio +rank0 [2026-11-15 03:42:06,990] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 03:42:06,990] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:45:59,875] INFO : Step 19000/500000 rank0 | Train Loss: 3.6684 | Train Loss Unweighted: 1.0602 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2522 | Regression_cos_phi: 0.0094 | Regression_energy: 0.3671 | Regression_eta: 0.0825 | Regression_pt: 0.3286 | Regression_sin_phi: 0.0199 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1244 | Classification_binary: 0.2522 | Regression_cos_phi: 0.5523 | Regression_energy: 0.2197 | Regression_eta: 1.0717 | Regression_pt: 0.1965 | Regression_sin_phi: 1.2515 | LR: 3.88e-05 | DataLoad Time: 0.0023s | Model Forward Time: 1.4804s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0023 | max_forward: 0.0550 | max_loss: 0.0029 | max_backward: 0.1179 | max_optimizer_step: 0.0718 | max_tensorboard_logging: 0.9567 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 6745.0 | uss_MB: 5535.9 | child_rss_MB: 23734.8 | allocated_MB: 195.1 | reserved_MB: 19920.0 | inactive_split_MB: 278.9 +rank0 [2026-11-15 03:45:59,878] INFO : Running validation on rank0 +rank0 [2026-11-15 03:45:59,879] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:45:59,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:46:00,495] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 03:46:00,616] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:46:00,733] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:46:00,803] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:46:00,870] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:46:00,935] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:46:01,001] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:46:01,069] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:46:01,175] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:46:01,276] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:46:01,377] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:46:01,444] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:46:01,510] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:46:01,574] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:46:01,640] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:46:01,704] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:46:01,804] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:46:01,811] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:46:01,814] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:46:01,816] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:46:01,819] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:46:01,822] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:46:01,824] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:46:01,826] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:46:01,829] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:46:01,831] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:46:01,833] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:46:01,836] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:46:01,838] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:46:01,841] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:46:01,843] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:46:01,846] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:46:09,638] INFO : VALIDATION | Step=19000/500000 | Train Loss=3.6684 | Valid Loss=2.8504 | Train Loss Unweighted=1.0602 | Valid Loss Unweighted=0.9862 | Stale=0 | ETA=2208.1m +rank0 [2026-11-15 03:46:09,638] INFO : split_configs=['1'] +rank0 [2026-11-15 03:46:09,682] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:46:09,689] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:46:09,804] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:10,650] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:11,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:11,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:11,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:12,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:12,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:12,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:13,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:13,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:13,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:14,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:14,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:14,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:15,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:15,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:15,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:15,766] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:46:15,769] INFO : split_configs=['1'] +rank0 [2026-11-15 03:46:15,780] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:46:15,785] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:46:15,839] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:16,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:17,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:17,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:18,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:18,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:19,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:19,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:19,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:20,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:20,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:21,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:21,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:22,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:22,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:23,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:23,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:23,595] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:46:23,600] INFO : split_configs=['1'] +rank0 [2026-11-15 03:46:23,611] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:46:23,615] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:46:23,671] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:24,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:25,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:25,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:25,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:26,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:26,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:27,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:27,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:27,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:28,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:28,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:29,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:29,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:30,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:30,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:30,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:30,915] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:46:30,922] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf +rank0 [2026-11-15 03:46:32,042] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_qq_pf +rank0 [2026-11-15 03:46:32,467] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:33,751] INFO : Plotted jets +rank0 [2026-11-15 03:46:35,211] INFO : Plotted jet ratio +rank0 [2026-11-15 03:46:35,221] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:46:36,800] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:46:37,235] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:38,567] INFO : Plotted jets +rank0 [2026-11-15 03:46:39,583] INFO : Plotted jet ratio +rank0 [2026-11-15 03:46:39,593] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:46:40,989] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_19000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:46:41,442] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:42,739] INFO : Plotted jets +rank0 [2026-11-15 03:46:43,721] INFO : Plotted jet ratio +rank0 [2026-11-15 03:48:40,112] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 03:48:40,112] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:37,524] INFO : Step 20000/500000 rank0 | Train Loss: 2.1480 | Train Loss Unweighted: 0.8316 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2504 | Regression_cos_phi: 0.0039 | Regression_energy: 0.2645 | Regression_eta: 0.0437 | Regression_pt: 0.2582 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1302 | Classification_binary: 0.2504 | Regression_cos_phi: 0.2268 | Regression_energy: 0.1583 | Regression_eta: 0.5675 | Regression_pt: 0.1544 | Regression_sin_phi: 0.6603 | LR: 4.07e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.4362s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0708 | max_loss: 0.0029 | max_backward: 0.1559 | max_optimizer_step: 0.0730 | max_tensorboard_logging: 0.8509 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 6865.8 | uss_MB: 5657.0 | child_rss_MB: 23594.8 | allocated_MB: 198.1 | reserved_MB: 20462.0 | inactive_split_MB: 281.9 +rank0 [2026-11-15 03:50:37,527] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 03:50:37,595] INFO : Running validation on rank0 +rank0 [2026-11-15 03:50:37,596] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:50:37,596] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:50:38,213] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 03:50:38,332] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:50:38,444] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:50:38,513] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:50:38,578] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:50:38,642] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:50:38,707] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:50:38,809] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:50:38,923] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:50:39,027] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:50:39,129] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:50:39,198] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:50:39,264] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:50:39,330] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:50:39,397] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:50:39,464] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:50:39,568] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:50:39,588] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:50:39,600] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:50:39,603] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:50:39,613] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:50:39,626] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:50:39,629] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:50:39,639] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:50:39,652] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:50:39,655] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:50:39,658] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:50:39,673] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:50:39,675] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:50:39,676] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:50:39,677] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:50:39,678] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:50:47,413] INFO : VALIDATION | Step=20000/500000 | Train Loss=2.1480 | Valid Loss=2.8273 | Train Loss Unweighted=0.8316 | Valid Loss Unweighted=0.9721 | Stale=0 | ETA=2204.4m +rank0 [2026-11-15 03:50:47,414] INFO : split_configs=['1'] +rank0 [2026-11-15 03:50:47,425] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:50:47,428] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:50:47,507] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:50:48,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:50:48,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:50:49,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:50:49,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:50:49,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:50:50,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:50:50,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:50:50,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:50:51,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:50:51,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:50:51,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:50:52,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:50:52,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:50:52,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:50:53,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:50:53,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:50:53,502] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 03:50:53,506] INFO : split_configs=['1'] +rank0 [2026-11-15 03:50:53,518] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:50:53,521] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:50:53,571] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:50:54,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:50:54,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:50:55,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:50:55,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:50:56,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:50:56,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:50:57,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:50:57,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:50:58,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:50:58,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:50:58,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:50:59,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:50:59,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:51:00,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:51:00,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:51:01,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:51:01,186] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:51:01,189] INFO : split_configs=['1'] +rank0 [2026-11-15 03:51:01,200] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:51:01,203] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:51:01,254] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:51:02,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:51:02,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:51:03,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:51:03,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:51:03,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:51:04,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:51:04,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:51:05,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:51:05,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:51:05,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:51:06,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:51:06,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:51:07,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:51:07,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:51:07,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:51:08,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:51:08,374] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 03:51:08,382] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf +rank0 [2026-11-15 03:51:09,498] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_qq_pf +rank0 [2026-11-15 03:51:09,926] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:11,631] INFO : Plotted jets +rank0 [2026-11-15 03:51:12,648] INFO : Plotted jet ratio +rank0 [2026-11-15 03:51:12,659] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:51:14,257] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:51:14,689] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:16,029] INFO : Plotted jets +rank0 [2026-11-15 03:51:17,047] INFO : Plotted jet ratio +rank0 [2026-11-15 03:51:17,057] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:51:18,459] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_20000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:51:18,901] INFO : Plotted number of elements +rank0 [2026-11-15 03:51:20,209] INFO : Plotted jets +rank0 [2026-11-15 03:51:21,188] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:14,175] INFO : Step 21000/500000 rank0 | Train Loss: 3.8062 | Train Loss Unweighted: 1.0392 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2480 | Regression_cos_phi: 0.0162 | Regression_energy: 0.3656 | Regression_eta: 0.0863 | Regression_pt: 0.3074 | Regression_sin_phi: 0.0153 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1214 | Classification_binary: 0.2480 | Regression_cos_phi: 0.9509 | Regression_energy: 0.2188 | Regression_eta: 1.1212 | Regression_pt: 0.1838 | Regression_sin_phi: 0.9621 | LR: 4.28e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.3954s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0576 | max_loss: 0.0027 | max_backward: 0.1256 | max_optimizer_step: 0.0731 | max_tensorboard_logging: 0.8598 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6848.6 | uss_MB: 5639.8 | child_rss_MB: 23765.4 | allocated_MB: 195.7 | reserved_MB: 18816.0 | inactive_split_MB: 234.3 +rank0 [2026-11-15 03:55:14,179] INFO : Running validation on rank0 +rank0 [2026-11-15 03:55:14,179] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:55:14,179] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:55:14,804] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 03:55:14,924] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:55:15,040] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:55:15,111] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:55:15,177] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:55:15,242] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:55:15,308] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:55:15,372] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:55:15,480] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:55:15,582] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:55:15,683] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:55:15,750] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:55:15,815] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:55:15,878] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:55:15,943] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:55:16,008] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:55:16,111] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:55:16,134] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:55:16,147] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:55:16,150] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:55:16,160] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:55:16,173] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:55:16,176] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:55:16,177] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:55:16,178] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:55:16,179] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:55:16,180] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:55:16,181] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:55:16,182] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:55:16,183] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:55:16,194] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:55:16,195] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:55:23,936] INFO : VALIDATION | Step=21000/500000 | Train Loss=3.8062 | Valid Loss=2.8064 | Train Loss Unweighted=1.0392 | Valid Loss Unweighted=0.9680 | Stale=0 | ETA=2200.2m +rank0 [2026-11-15 03:55:23,936] INFO : split_configs=['1'] +rank0 [2026-11-15 03:55:23,947] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 03:55:23,962] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 03:55:24,055] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:55:24,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 03:55:25,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 03:55:25,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 03:55:25,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 03:55:26,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 03:55:26,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 03:55:26,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 03:55:27,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 03:55:27,639] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 03:55:27,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 03:55:28,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 03:55:28,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 03:55:28,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 03:55:29,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 03:55:29,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 03:55:29,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 03:55:30,034] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 03:55:30,037] INFO : split_configs=['1'] +rank0 [2026-11-15 03:55:30,048] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 03:55:30,054] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 03:55:30,112] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:55:31,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:55:31,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:55:32,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:55:32,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:55:32,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:55:33,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:55:33,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:55:34,226] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:55:34,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:55:35,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:55:35,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:55:36,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:55:36,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:55:36,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:55:37,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:55:37,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:55:37,854] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:55:37,857] INFO : split_configs=['1'] +rank0 [2026-11-15 03:55:37,868] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 03:55:37,872] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:55:37,931] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:55:38,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 03:55:39,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 03:55:39,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 03:55:40,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 03:55:40,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 03:55:41,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 03:55:41,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 03:55:41,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 03:55:42,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 03:55:42,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 03:55:43,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 03:55:43,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 03:55:43,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 03:55:44,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 03:55:44,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 03:55:45,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 03:55:45,161] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 03:55:45,170] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf +rank0 [2026-11-15 03:55:46,282] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_qq_pf +rank0 [2026-11-15 03:55:46,716] INFO : Plotted number of elements +rank0 [2026-11-15 03:55:47,982] INFO : Plotted jets +rank0 [2026-11-15 03:55:49,430] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:49,443] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:55:50,998] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ttbar_pf +rank0 [2026-11-15 03:55:51,449] INFO : Plotted number of elements +rank0 [2026-11-15 03:55:52,767] INFO : Plotted jets +rank0 [2026-11-15 03:55:53,788] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:53,808] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:55:55,211] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_21000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 03:55:55,662] INFO : Plotted number of elements +rank0 [2026-11-15 03:55:56,982] INFO : Plotted jets +rank0 [2026-11-15 03:55:57,975] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:58,381] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 03:55:58,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:59:51,752] INFO : Step 22000/500000 rank0 | Train Loss: 1.5651 | Train Loss Unweighted: 0.8086 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2504 | Regression_cos_phi: 0.0044 | Regression_energy: 0.2623 | Regression_eta: 0.0227 | Regression_pt: 0.2632 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1272 | Classification_binary: 0.2504 | Regression_cos_phi: 0.2581 | Regression_energy: 0.1570 | Regression_eta: 0.2943 | Regression_pt: 0.1574 | Regression_sin_phi: 0.3206 | LR: 4.50e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4057s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0544 | max_loss: 0.0029 | max_backward: 0.1171 | max_optimizer_step: 0.0735 | max_tensorboard_logging: 0.8662 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 6925.9 | uss_MB: 5717.1 | child_rss_MB: 23679.8 | allocated_MB: 196.4 | reserved_MB: 16168.0 | inactive_split_MB: 247.6 +rank0 [2026-11-15 03:59:51,754] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 03:59:51,821] INFO : Running validation on rank0 +rank0 [2026-11-15 03:59:51,821] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:59:51,821] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:59:52,445] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 03:59:52,564] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:59:52,677] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:59:52,746] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:59:52,813] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:59:52,879] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:59:52,946] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:59:53,012] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:59:53,122] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:59:53,225] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:59:53,327] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:59:53,394] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:59:53,461] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:59:53,527] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:59:53,593] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:59:53,660] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:59:53,763] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:59:53,782] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:59:53,795] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:59:53,798] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:59:53,808] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:59:53,819] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:59:53,821] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:59:53,823] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:59:53,826] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:59:53,828] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:59:53,836] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:59:53,840] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:59:53,842] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:59:53,845] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:59:53,848] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:59:53,850] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:00:01,548] INFO : VALIDATION | Step=22000/500000 | Train Loss=1.5651 | Valid Loss=2.8230 | Train Loss Unweighted=0.8086 | Valid Loss Unweighted=0.9650 | Stale=1 | ETA=2196.3m +rank0 [2026-11-15 04:00:01,548] INFO : split_configs=['1'] +rank0 [2026-11-15 04:00:01,559] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:00:01,566] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:00:01,650] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:02,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:02,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:03,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:03,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:04,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:04,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:04,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:05,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:05,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:05,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:06,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:06,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:06,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:07,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:07,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:07,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:07,793] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:00:07,797] INFO : split_configs=['1'] +rank0 [2026-11-15 04:00:07,808] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:00:07,811] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:00:07,869] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:08,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:09,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:09,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:10,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:10,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:11,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:11,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:11,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:12,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:12,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:13,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:13,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:14,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:14,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:15,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:15,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:15,635] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:00:15,638] INFO : split_configs=['1'] +rank0 [2026-11-15 04:00:15,649] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:00:15,652] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:00:15,711] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:16,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:17,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:17,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:17,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:18,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:18,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:19,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:19,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:19,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:20,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:20,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:21,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:21,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:22,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:22,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:22,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:22,930] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:00:22,938] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf +rank0 [2026-11-15 04:00:24,077] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_qq_pf +rank0 [2026-11-15 04:00:24,507] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:26,225] INFO : Plotted jets +rank0 [2026-11-15 04:00:27,260] INFO : Plotted jet ratio +rank0 [2026-11-15 04:00:27,271] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:00:28,860] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:00:29,306] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:30,646] INFO : Plotted jets +rank0 [2026-11-15 04:00:31,671] INFO : Plotted jet ratio +rank0 [2026-11-15 04:00:31,682] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:00:33,114] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_22000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:00:33,569] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:34,903] INFO : Plotted jets +rank0 [2026-11-15 04:00:35,888] INFO : Plotted jet ratio +rank0 [2026-11-15 04:02:32,421] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 04:02:32,421] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:04:30,177] INFO : Step 23000/500000 rank0 | Train Loss: 2.4732 | Train Loss Unweighted: 0.9867 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2583 | Regression_cos_phi: 0.0069 | Regression_energy: 0.3323 | Regression_eta: 0.0427 | Regression_pt: 0.3340 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0934 | Classification_binary: 0.2583 | Regression_cos_phi: 0.4078 | Regression_energy: 0.1988 | Regression_eta: 0.5545 | Regression_pt: 0.1998 | Regression_sin_phi: 0.7605 | LR: 4.73e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4812s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0594 | max_loss: 0.0029 | max_backward: 0.1240 | max_optimizer_step: 0.0760 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7100.6 | uss_MB: 5891.8 | child_rss_MB: 23684.4 | allocated_MB: 197.0 | reserved_MB: 18782.0 | inactive_split_MB: 259.0 +rank0 [2026-11-15 04:04:30,179] INFO : Running validation on rank0 +rank0 [2026-11-15 04:04:30,179] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:04:30,179] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:04:30,812] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:04:30,939] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:04:31,054] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:04:31,125] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:04:31,191] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:04:31,257] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:04:31,323] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:04:31,391] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:04:31,497] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:04:31,603] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:04:31,708] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:04:31,778] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:04:31,846] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:04:31,914] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:04:31,982] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:04:32,051] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:04:32,157] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:04:32,177] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:04:32,189] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:04:32,193] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:04:32,202] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:04:32,215] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:04:32,218] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:04:32,228] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:04:32,241] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:04:32,244] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:04:32,246] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:04:32,249] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:04:32,251] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:04:32,253] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:04:32,256] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:04:32,258] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:04:39,974] INFO : VALIDATION | Step=23000/500000 | Train Loss=2.4732 | Valid Loss=2.7669 | Train Loss Unweighted=0.9867 | Valid Loss Unweighted=0.9500 | Stale=0 | ETA=2192.7m +rank0 [2026-11-15 04:04:39,974] INFO : split_configs=['1'] +rank0 [2026-11-15 04:04:39,985] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:04:39,990] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:04:40,070] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:40,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:41,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:41,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:41,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:42,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:42,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:42,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:43,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:43,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:43,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:44,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:44,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:44,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:04:45,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:04:45,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:04:45,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:04:46,057] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:04:46,060] INFO : split_configs=['1'] +rank0 [2026-11-15 04:04:46,071] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:04:46,074] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:04:46,125] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:47,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:47,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:47,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:48,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:48,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:49,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:49,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:50,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:50,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:51,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:51,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:51,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:52,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:04:52,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:04:53,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:04:53,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:04:53,791] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:04:53,794] INFO : split_configs=['1'] +rank0 [2026-11-15 04:04:53,805] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:04:53,809] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:04:53,861] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:54,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:55,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:55,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:56,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:56,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:56,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:57,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:57,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:58,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:58,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:58,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:59,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:59,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:05:00,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:05:00,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:05:00,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:05:01,017] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:05:01,025] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf +rank0 [2026-11-15 04:05:02,176] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_qq_pf +rank0 [2026-11-15 04:05:02,611] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:03,885] INFO : Plotted jets +rank0 [2026-11-15 04:05:05,363] INFO : Plotted jet ratio +rank0 [2026-11-15 04:05:05,373] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:05:06,981] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:05:07,419] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:08,765] INFO : Plotted jets +rank0 [2026-11-15 04:05:09,791] INFO : Plotted jet ratio +rank0 [2026-11-15 04:05:09,801] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:05:11,213] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_23000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:05:11,653] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:12,970] INFO : Plotted jets +rank0 [2026-11-15 04:05:13,950] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:07,195] INFO : Step 24000/500000 rank0 | Train Loss: 3.6223 | Train Loss Unweighted: 1.0212 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2495 | Regression_cos_phi: 0.0183 | Regression_energy: 0.3425 | Regression_eta: 0.0739 | Regression_pt: 0.3233 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0999 | Classification_binary: 0.2495 | Regression_cos_phi: 1.0729 | Regression_energy: 0.2050 | Regression_eta: 0.9600 | Regression_pt: 0.1934 | Regression_sin_phi: 0.8418 | LR: 4.97e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4404s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0599 | max_loss: 0.0027 | max_backward: 0.1265 | max_optimizer_step: 0.0755 | max_tensorboard_logging: 0.8883 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7142.8 | uss_MB: 5933.9 | child_rss_MB: 23798.4 | allocated_MB: 197.5 | reserved_MB: 20284.0 | inactive_split_MB: 288.5 +rank0 [2026-11-15 04:09:07,197] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:09:07,261] INFO : Running validation on rank0 +rank0 [2026-11-15 04:09:07,261] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:09:07,261] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:09:07,883] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:09:08,004] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:09:08,124] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:09:08,194] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:09:08,262] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:09:08,329] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:09:08,395] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:09:08,462] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:09:08,569] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:09:08,671] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:09:08,772] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:09:08,839] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:09:08,905] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:09:08,969] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:09:09,035] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:09:09,101] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:09:09,201] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:09:09,208] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:09:09,211] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:09:09,213] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:09:09,216] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:09:09,219] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:09:09,221] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:09:09,223] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:09:09,226] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:09:09,228] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:09:09,231] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:09:09,233] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:09:09,236] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:09:09,238] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:09:09,241] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:09:09,243] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:09:16,955] INFO : VALIDATION | Step=24000/500000 | Train Loss=3.6223 | Valid Loss=2.7635 | Train Loss Unweighted=1.0212 | Valid Loss Unweighted=0.9409 | Stale=0 | ETA=2188.5m +rank0 [2026-11-15 04:09:16,955] INFO : split_configs=['1'] +rank0 [2026-11-15 04:09:16,966] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:09:16,969] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:09:17,061] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:09:17,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:09:18,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:09:18,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:09:18,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:09:19,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:09:19,647] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:09:19,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:09:20,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:09:20,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:09:20,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:09:21,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:09:21,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:09:22,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:09:22,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:09:22,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:09:22,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:09:23,053] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:09:23,056] INFO : split_configs=['1'] +rank0 [2026-11-15 04:09:23,067] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:09:23,074] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:09:23,143] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:09:24,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:09:24,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:09:25,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:09:25,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:09:25,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:09:26,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:09:26,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:09:27,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:09:27,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:09:28,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:09:28,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:09:29,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:09:29,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:09:29,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:09:30,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:09:30,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:09:30,927] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:09:30,930] INFO : split_configs=['1'] +rank0 [2026-11-15 04:09:30,940] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:09:30,943] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:09:31,004] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:09:32,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:09:32,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:09:32,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:09:33,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:09:33,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:09:34,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:09:34,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:09:34,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:09:35,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:09:35,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:09:36,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:09:36,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:09:36,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:09:37,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:09:37,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:09:38,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:09:38,252] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:09:38,259] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf +rank0 [2026-11-15 04:09:39,402] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_qq_pf +rank0 [2026-11-15 04:09:39,827] INFO : Plotted number of elements +rank0 [2026-11-15 04:09:41,100] INFO : Plotted jets +rank0 [2026-11-15 04:09:42,550] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:42,563] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:09:44,125] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:09:44,570] INFO : Plotted number of elements +rank0 [2026-11-15 04:09:45,896] INFO : Plotted jets +rank0 [2026-11-15 04:09:46,914] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:46,923] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:09:48,323] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_24000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:09:48,765] INFO : Plotted number of elements +rank0 [2026-11-15 04:09:50,072] INFO : Plotted jets +rank0 [2026-11-15 04:09:51,053] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:51,460] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:09:51,460] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:13:44,448] INFO : Step 25000/500000 rank0 | Train Loss: 2.7232 | Train Loss Unweighted: 0.9901 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2429 | Regression_cos_phi: 0.0136 | Regression_energy: 0.3340 | Regression_eta: 0.0302 | Regression_pt: 0.3566 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2429 | Regression_cos_phi: 0.7996 | Regression_energy: 0.1999 | Regression_eta: 0.3929 | Regression_pt: 0.2133 | Regression_sin_phi: 0.7766 | LR: 5.22e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.5365s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0022 | max_forward: 0.0542 | max_loss: 0.0029 | max_backward: 0.1154 | max_optimizer_step: 0.0755 | max_tensorboard_logging: 0.9920 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 7240.9 | uss_MB: 6032.1 | child_rss_MB: 23713.7 | allocated_MB: 192.6 | reserved_MB: 18926.0 | inactive_split_MB: 171.4 +rank0 [2026-11-15 04:13:44,450] INFO : Running validation on rank0 +rank0 [2026-11-15 04:13:44,450] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:13:44,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:13:45,072] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:13:45,194] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:13:45,306] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:13:45,376] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:13:45,442] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:13:45,508] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:13:45,574] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:13:45,640] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:13:45,747] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:13:45,853] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:13:45,957] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:13:46,027] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:13:46,095] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:13:46,159] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:13:46,225] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:13:46,290] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:13:46,392] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:13:46,399] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:13:46,402] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:13:46,404] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:13:46,407] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:13:46,410] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:13:46,412] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:13:46,415] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:13:46,417] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:13:46,420] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:13:46,422] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:13:46,425] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:13:46,427] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:13:46,429] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:13:46,432] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:13:46,434] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:13:54,093] INFO : VALIDATION | Step=25000/500000 | Train Loss=2.7232 | Valid Loss=2.7764 | Train Loss Unweighted=0.9901 | Valid Loss Unweighted=0.9439 | Stale=1 | ETA=2184.3m +rank0 [2026-11-15 04:13:54,094] INFO : split_configs=['1'] +rank0 [2026-11-15 04:13:54,127] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:13:54,133] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:13:54,256] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:13:55,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:13:55,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:13:55,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:13:56,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:13:56,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:13:56,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:13:57,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:13:57,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:13:57,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:13:58,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:13:58,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:13:58,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:13:59,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:13:59,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:13:59,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:00,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:00,271] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:14:00,274] INFO : split_configs=['1'] +rank0 [2026-11-15 04:14:00,305] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:14:00,308] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:14:00,368] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:14:01,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:14:01,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:14:02,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:14:02,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:14:03,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:14:03,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:14:04,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:14:04,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:14:04,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:14:05,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:14:05,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:14:06,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:14:06,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:14:07,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:14:07,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:08,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:08,153] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:14:08,156] INFO : split_configs=['1'] +rank0 [2026-11-15 04:14:08,205] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:14:08,209] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:14:08,275] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:14:09,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:14:09,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:14:10,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:14:10,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:14:10,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:14:11,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:14:11,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:14:12,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:14:12,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:14:12,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:14:13,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:14:13,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:14:14,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:14:14,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:14:15,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:15,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:15,534] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:14:15,543] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf +rank0 [2026-11-15 04:14:16,684] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_qq_pf +rank0 [2026-11-15 04:14:17,112] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:18,417] INFO : Plotted jets +rank0 [2026-11-15 04:14:19,896] INFO : Plotted jet ratio +rank0 [2026-11-15 04:14:19,908] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:14:21,496] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:14:21,951] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:23,291] INFO : Plotted jets +rank0 [2026-11-15 04:14:24,324] INFO : Plotted jet ratio +rank0 [2026-11-15 04:14:24,333] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:14:25,747] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_25000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:14:26,199] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:27,535] INFO : Plotted jets +rank0 [2026-11-15 04:14:28,560] INFO : Plotted jet ratio +rank0 [2026-11-15 04:16:25,059] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:16:25,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:22,664] INFO : Step 26000/500000 rank0 | Train Loss: 2.4374 | Train Loss Unweighted: 0.9093 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2384 | Regression_cos_phi: 0.0066 | Regression_energy: 0.3054 | Regression_eta: 0.0388 | Regression_pt: 0.3064 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.2384 | Regression_cos_phi: 0.3901 | Regression_energy: 0.1828 | Regression_eta: 0.5042 | Regression_pt: 0.1833 | Regression_sin_phi: 0.8374 | LR: 5.47e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4747s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0640 | max_loss: 0.0040 | max_backward: 0.1365 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.8381 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7251.5 | uss_MB: 6042.7 | child_rss_MB: 23874.5 | allocated_MB: 198.2 | reserved_MB: 16156.0 | inactive_split_MB: 149.8 +rank0 [2026-11-15 04:18:22,669] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:18:22,737] INFO : Running validation on rank0 +rank0 [2026-11-15 04:18:22,737] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:18:22,737] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:18:23,369] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:18:23,495] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:18:23,608] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:18:23,679] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:18:23,745] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:18:23,814] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:18:23,880] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:18:23,948] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:18:24,059] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:18:24,168] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:18:24,276] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:18:24,344] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:18:24,412] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:18:24,479] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:18:24,545] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:18:24,612] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:18:24,714] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:18:24,721] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:18:24,724] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:18:24,726] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:18:24,729] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:18:24,732] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:18:24,734] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:18:24,736] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:18:24,739] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:18:24,742] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:18:24,744] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:18:24,747] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:18:24,749] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:18:24,751] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:18:24,754] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:18:24,756] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:18:32,524] INFO : VALIDATION | Step=26000/500000 | Train Loss=2.4374 | Valid Loss=2.7336 | Train Loss Unweighted=0.9093 | Valid Loss Unweighted=0.9298 | Stale=0 | ETA=2180.4m +rank0 [2026-11-15 04:18:32,525] INFO : split_configs=['1'] +rank0 [2026-11-15 04:18:32,536] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:18:32,548] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:18:32,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:33,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:33,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:34,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:34,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:34,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:35,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:35,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:35,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:36,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:36,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:36,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:37,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:37,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:37,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:38,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:38,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:38,572] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:18:38,575] INFO : split_configs=['1'] +rank0 [2026-11-15 04:18:38,586] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:18:38,589] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:18:38,641] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:39,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:40,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:40,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:40,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:41,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:41,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:42,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:42,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:43,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:43,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:44,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:44,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:44,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:45,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:45,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:46,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:46,370] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:18:46,373] INFO : split_configs=['1'] +rank0 [2026-11-15 04:18:46,384] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:18:46,389] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:18:46,447] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:47,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:47,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:48,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:48,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:49,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:49,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:49,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:50,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:50,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:51,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:51,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:51,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:52,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:52,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:53,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:53,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:53,575] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 04:18:53,584] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf +rank0 [2026-11-15 04:18:54,717] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_qq_pf +rank0 [2026-11-15 04:18:55,156] INFO : Plotted number of elements +rank0 [2026-11-15 04:18:56,882] INFO : Plotted jets +rank0 [2026-11-15 04:18:57,901] INFO : Plotted jet ratio +rank0 [2026-11-15 04:18:57,914] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:18:59,526] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:18:59,956] INFO : Plotted number of elements +rank0 [2026-11-15 04:19:01,299] INFO : Plotted jets +rank0 [2026-11-15 04:19:02,322] INFO : Plotted jet ratio +rank0 [2026-11-15 04:19:02,332] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:19:03,741] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_26000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:19:04,202] INFO : Plotted number of elements +rank0 [2026-11-15 04:19:05,512] INFO : Plotted jets +rank0 [2026-11-15 04:19:06,505] INFO : Plotted jet ratio +rank0 [2026-11-15 04:22:59,715] INFO : Step 27000/500000 rank0 | Train Loss: 2.2512 | Train Loss Unweighted: 0.9266 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0079 | Regression_energy: 0.3235 | Regression_eta: 0.0177 | Regression_pt: 0.3335 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2301 | Regression_cos_phi: 0.4662 | Regression_energy: 0.1936 | Regression_eta: 0.2299 | Regression_pt: 0.1994 | Regression_sin_phi: 0.8566 | LR: 5.74e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4271s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0029 | max_forward: 0.0637 | max_loss: 0.0028 | max_backward: 0.1336 | max_optimizer_step: 0.0755 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7396.7 | uss_MB: 6187.9 | child_rss_MB: 23664.2 | allocated_MB: 199.2 | reserved_MB: 18332.0 | inactive_split_MB: 322.8 +rank0 [2026-11-15 04:22:59,719] INFO : Running validation on rank0 +rank0 [2026-11-15 04:22:59,719] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:22:59,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:23:00,347] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:23:00,469] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:23:00,584] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:23:00,656] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:23:00,724] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:23:00,790] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:23:00,856] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:23:00,922] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:23:01,029] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:23:01,135] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:23:01,237] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:23:01,303] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:23:01,368] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:23:01,433] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:23:01,499] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:23:01,564] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:23:01,664] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:23:01,671] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:23:01,674] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:23:01,677] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:23:01,679] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:23:01,682] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:23:01,684] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:23:01,687] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:23:01,689] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:23:01,692] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:23:01,694] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:23:01,697] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:23:01,699] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:23:01,701] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:23:01,704] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:23:01,706] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:23:09,413] INFO : VALIDATION | Step=27000/500000 | Train Loss=2.2512 | Valid Loss=2.7552 | Train Loss Unweighted=0.9266 | Valid Loss Unweighted=0.9283 | Stale=1 | ETA=2176.1m +rank0 [2026-11-15 04:23:09,413] INFO : split_configs=['1'] +rank0 [2026-11-15 04:23:09,426] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:23:09,433] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:23:09,521] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:10,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:10,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:11,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:11,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:11,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:12,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:12,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:12,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:13,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:13,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:13,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:14,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:14,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:14,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:15,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:15,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:15,597] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:23:15,600] INFO : split_configs=['1'] +rank0 [2026-11-15 04:23:15,612] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:23:15,615] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:23:15,674] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:16,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:17,226] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:17,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:18,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:18,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:18,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:19,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:19,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:20,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:20,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:21,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:21,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:22,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:22,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:22,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:23,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:23,483] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:23:23,485] INFO : split_configs=['1'] +rank0 [2026-11-15 04:23:23,497] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:23:23,501] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:23:23,566] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:24,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:25,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:25,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:25,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:26,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:26,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:27,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:27,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:27,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:28,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:28,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:29,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:29,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:29,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:30,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:30,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:30,863] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:23:30,870] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf +rank0 [2026-11-15 04:23:32,015] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_qq_pf +rank0 [2026-11-15 04:23:32,442] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:33,733] INFO : Plotted jets +rank0 [2026-11-15 04:23:35,222] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:35,234] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:23:36,834] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:23:37,273] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:38,614] INFO : Plotted jets +rank0 [2026-11-15 04:23:39,646] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:39,655] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:23:41,069] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_27000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:23:41,523] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:42,842] INFO : Plotted jets +rank0 [2026-11-15 04:23:43,838] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:44,259] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:23:44,260] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:37,525] INFO : Step 28000/500000 rank0 | Train Loss: 2.1634 | Train Loss Unweighted: 0.8074 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2405 | Regression_cos_phi: 0.0134 | Regression_energy: 0.2678 | Regression_eta: 0.0223 | Regression_pt: 0.2559 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.2405 | Regression_cos_phi: 0.7856 | Regression_energy: 0.1603 | Regression_eta: 0.2892 | Regression_pt: 0.1530 | Regression_sin_phi: 0.4535 | LR: 6.01e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4176s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0030 | max_forward: 0.0651 | max_loss: 0.0029 | max_backward: 0.1380 | max_optimizer_step: 0.0765 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7403.7 | uss_MB: 6194.9 | child_rss_MB: 23650.1 | allocated_MB: 200.6 | reserved_MB: 17570.0 | inactive_split_MB: 151.4 +rank0 [2026-11-15 04:27:37,528] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 04:27:37,596] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:37,596] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:37,596] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:38,219] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 04:27:38,338] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:38,449] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:38,518] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:38,584] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:38,648] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:38,714] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:38,780] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:38,889] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:38,994] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:39,101] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:39,170] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:39,237] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:39,304] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:39,370] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:39,436] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:39,537] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:39,544] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:39,546] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:39,549] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:39,551] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:39,554] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:39,556] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:39,559] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:39,561] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:39,564] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:39,566] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:39,569] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:39,571] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:39,573] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:39,576] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:39,578] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:47,285] INFO : VALIDATION | Step=28000/500000 | Train Loss=2.1634 | Valid Loss=2.7090 | Train Loss Unweighted=0.8074 | Valid Loss Unweighted=0.9227 | Stale=0 | ETA=2172.0m +rank0 [2026-11-15 04:27:47,287] INFO : split_configs=['1'] +rank0 [2026-11-15 04:27:47,300] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:27:47,308] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:27:47,396] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:48,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:48,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:48,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:49,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:49,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:49,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:50,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:50,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:50,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:51,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:51,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:52,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:52,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:52,647] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:52,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:53,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:53,375] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:27:53,378] INFO : split_configs=['1'] +rank0 [2026-11-15 04:27:53,389] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:27:53,392] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:27:53,450] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:54,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:54,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:55,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:55,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:56,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:56,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:57,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:57,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:58,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:58,465] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:58,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:59,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:59,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:00,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:00,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:01,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:01,210] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:28:01,215] INFO : split_configs=['1'] +rank0 [2026-11-15 04:28:01,225] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:28:01,229] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:28:01,290] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:28:02,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:28:02,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:28:03,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:28:03,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:28:03,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:28:04,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:28:04,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:28:05,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:28:05,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:28:05,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:28:06,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:28:06,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:28:07,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:07,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:08,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:08,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:08,513] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:28:08,520] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf +rank0 [2026-11-15 04:28:09,658] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_qq_pf +rank0 [2026-11-15 04:28:10,086] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:11,832] INFO : Plotted jets +rank0 [2026-11-15 04:28:12,860] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:12,872] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:28:14,467] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:28:14,912] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:16,254] INFO : Plotted jets +rank0 [2026-11-15 04:28:17,278] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:17,287] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:28:18,748] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_28000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:28:19,207] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:20,525] INFO : Plotted jets +rank0 [2026-11-15 04:28:21,506] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:17,206] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:30:17,207] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:13,934] INFO : Step 29000/500000 rank0 | Train Loss: 2.3262 | Train Loss Unweighted: 0.8439 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2280 | Regression_cos_phi: 0.0126 | Regression_energy: 0.2810 | Regression_eta: 0.0211 | Regression_pt: 0.2906 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2280 | Regression_cos_phi: 0.7386 | Regression_energy: 0.1682 | Regression_eta: 0.2741 | Regression_pt: 0.1738 | Regression_sin_phi: 0.6456 | LR: 6.29e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.4801s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0612 | max_loss: 0.0029 | max_backward: 0.1305 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.9008 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 5804.6 | uss_MB: 4595.8 | child_rss_MB: 23672.4 | allocated_MB: 194.9 | reserved_MB: 18540.0 | inactive_split_MB: 287.1 +rank0 [2026-11-15 04:32:13,936] INFO : Running validation on rank0 +rank0 [2026-11-15 04:32:13,936] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:32:13,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:32:14,567] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 04:32:14,692] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:32:14,804] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:32:14,874] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:32:14,941] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:32:15,006] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:32:15,074] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:32:15,141] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:32:15,249] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:32:15,355] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:32:15,462] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:32:15,530] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:32:15,597] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:32:15,665] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:32:15,732] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:32:15,799] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:32:15,900] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:32:15,907] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:32:15,910] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:32:15,912] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:32:15,915] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:32:15,918] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:32:15,920] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:32:15,922] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:32:15,926] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:32:15,929] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:32:15,931] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:32:15,934] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:32:15,937] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:32:15,939] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:32:15,942] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:32:15,944] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:32:23,616] INFO : VALIDATION | Step=29000/500000 | Train Loss=2.3262 | Valid Loss=2.7383 | Train Loss Unweighted=0.8439 | Valid Loss Unweighted=0.9146 | Stale=1 | ETA=2167.5m +rank0 [2026-11-15 04:32:23,616] INFO : split_configs=['1'] +rank0 [2026-11-15 04:32:23,627] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:32:23,632] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:32:23,714] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:24,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:24,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:25,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:25,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:25,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:26,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:26,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:26,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:27,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:27,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:27,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:28,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:28,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:28,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:29,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:29,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:29,685] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:32:29,688] INFO : split_configs=['1'] +rank0 [2026-11-15 04:32:29,738] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:32:29,741] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:32:29,794] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:30,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:31,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:31,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:32,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:32,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:32,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:33,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:33,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:34,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:34,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:35,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:35,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:36,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:36,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:36,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:37,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:37,478] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:32:37,482] INFO : split_configs=['1'] +rank0 [2026-11-15 04:32:37,532] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:32:37,536] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:32:37,596] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:38,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:38,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:39,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:39,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:40,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:40,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:41,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:41,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:41,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:42,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:42,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:43,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:43,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:43,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:44,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:44,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:44,767] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 04:32:44,774] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf +rank0 [2026-11-15 04:32:45,924] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_qq_pf +rank0 [2026-11-15 04:32:46,360] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:47,629] INFO : Plotted jets +rank0 [2026-11-15 04:32:49,098] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:49,109] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:32:50,738] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:32:51,172] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:52,496] INFO : Plotted jets +rank0 [2026-11-15 04:32:53,515] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:53,528] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:32:54,931] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_29000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:32:55,374] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:56,679] INFO : Plotted jets +rank0 [2026-11-15 04:32:57,664] INFO : Plotted jet ratio +rank0 [2026-11-15 04:36:51,240] INFO : Step 30000/500000 rank0 | Train Loss: 1.9500 | Train Loss Unweighted: 0.7760 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2434 | Regression_cos_phi: 0.0097 | Regression_energy: 0.2483 | Regression_eta: 0.0239 | Regression_pt: 0.2431 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2434 | Regression_cos_phi: 0.5689 | Regression_energy: 0.1486 | Regression_eta: 0.3108 | Regression_pt: 0.1454 | Regression_sin_phi: 0.4574 | LR: 6.58e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.5529s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0636 | max_loss: 0.0027 | max_backward: 0.1340 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.9691 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 7609.4 | uss_MB: 6400.6 | child_rss_MB: 23780.2 | allocated_MB: 198.4 | reserved_MB: 24478.0 | inactive_split_MB: 353.6 +rank0 [2026-11-15 04:36:51,242] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 04:36:51,308] INFO : Running validation on rank0 +rank0 [2026-11-15 04:36:51,308] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:36:51,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:36:51,934] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 04:36:52,056] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:36:52,175] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:36:52,246] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:36:52,312] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:36:52,377] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:36:52,443] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:36:52,510] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:36:52,618] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:36:52,722] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:36:52,825] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:36:52,892] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:36:52,958] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:36:53,023] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:36:53,091] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:36:53,157] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:36:53,259] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:36:53,282] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:36:53,295] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:36:53,298] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:36:53,308] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:36:53,320] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:36:53,324] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:36:53,365] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:36:53,368] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:36:53,370] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:36:53,372] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:36:53,375] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:36:53,377] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:36:53,380] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:36:53,383] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:36:53,385] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:01,052] INFO : VALIDATION | Step=30000/500000 | Train Loss=1.9500 | Valid Loss=2.7121 | Train Loss Unweighted=0.7760 | Valid Loss Unweighted=0.9101 | Stale=2 | ETA=2163.2m +rank0 [2026-11-15 04:37:01,052] INFO : split_configs=['1'] +rank0 [2026-11-15 04:37:01,093] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:37:01,100] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:37:01,199] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:02,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:02,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:02,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:03,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:03,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:03,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:04,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:04,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:04,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:05,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:05,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:05,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:06,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:06,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:06,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:07,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:07,233] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:37:07,236] INFO : split_configs=['1'] +rank0 [2026-11-15 04:37:07,247] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:37:07,252] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:37:07,313] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:08,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:08,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:09,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:09,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:10,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:10,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:10,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:11,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:11,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:12,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:12,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:13,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:13,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:14,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:14,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:15,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:15,083] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:37:15,087] INFO : split_configs=['1'] +rank0 [2026-11-15 04:37:15,098] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:37:15,101] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:37:15,162] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:16,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:16,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:17,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:17,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:17,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:18,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:18,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:19,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:19,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:19,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:20,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:20,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:21,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:21,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:21,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:22,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:22,410] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:37:22,418] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf +rank0 [2026-11-15 04:37:23,546] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_qq_pf +rank0 [2026-11-15 04:37:23,965] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:25,232] INFO : Plotted jets +rank0 [2026-11-15 04:37:26,708] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:26,721] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:37:28,340] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:37:28,775] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:30,109] INFO : Plotted jets +rank0 [2026-11-15 04:37:31,132] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:31,142] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:37:32,535] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_30000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:37:32,975] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:34,288] INFO : Plotted jets +rank0 [2026-11-15 04:37:35,273] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:35,693] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:37:35,693] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:29,265] INFO : Step 31000/500000 rank0 | Train Loss: 3.0354 | Train Loss Unweighted: 1.0306 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2339 | Regression_cos_phi: 0.0087 | Regression_energy: 0.3662 | Regression_eta: 0.0801 | Regression_pt: 0.3297 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1029 | Classification_binary: 0.2339 | Regression_cos_phi: 0.5129 | Regression_energy: 0.2191 | Regression_eta: 1.0412 | Regression_pt: 0.1972 | Regression_sin_phi: 0.7282 | LR: 6.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.5941s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0559 | max_loss: 0.0030 | max_backward: 0.1172 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 1.0237 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7607.0 | uss_MB: 6461.5 | child_rss_MB: 23805.7 | allocated_MB: 194.6 | reserved_MB: 22638.0 | inactive_split_MB: 297.4 +rank0 [2026-11-15 04:41:29,269] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:29,269] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:29,269] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:29,901] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 04:41:30,024] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:30,143] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:30,214] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:30,280] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:30,345] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:30,412] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:30,479] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:30,587] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:30,692] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:30,793] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:30,862] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:30,928] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:30,993] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:31,061] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:31,128] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:31,230] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:31,253] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:31,266] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:31,269] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:31,279] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:31,292] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:31,295] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:31,305] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:31,318] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:31,321] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:31,331] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:31,340] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:31,342] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:31,344] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:31,347] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:31,350] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:39,025] INFO : VALIDATION | Step=31000/500000 | Train Loss=3.0354 | Valid Loss=2.7676 | Train Loss Unweighted=1.0306 | Valid Loss Unweighted=0.9194 | Stale=3 | ETA=2159.1m +rank0 [2026-11-15 04:41:39,026] INFO : split_configs=['1'] +rank0 [2026-11-15 04:41:39,073] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:41:39,081] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:41:39,173] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:40,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:40,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:40,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:41,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:41,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:41,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:42,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:42,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:42,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:43,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:43,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:43,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:44,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:44,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:44,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:45,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:45,221] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:41:45,224] INFO : split_configs=['1'] +rank0 [2026-11-15 04:41:45,265] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:41:45,269] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:41:45,330] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:46,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:46,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:47,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:47,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:48,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:48,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:48,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:49,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:49,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:50,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:50,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:51,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:51,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:52,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:52,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:53,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:53,079] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:41:53,083] INFO : split_configs=['1'] +rank0 [2026-11-15 04:41:53,121] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:41:53,125] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:41:53,187] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:54,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:54,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:55,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:55,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:55,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:56,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:56,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:57,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:57,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:57,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:58,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:58,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:59,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:59,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:59,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:42:00,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:42:00,414] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:42:00,421] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf +rank0 [2026-11-15 04:42:01,577] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_qq_pf +rank0 [2026-11-15 04:42:02,004] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:03,301] INFO : Plotted jets +rank0 [2026-11-15 04:42:04,800] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:04,812] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:42:06,396] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:42:06,831] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:08,181] INFO : Plotted jets +rank0 [2026-11-15 04:42:09,202] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:09,211] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:42:10,618] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_31000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:42:11,086] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:12,396] INFO : Plotted jets +rank0 [2026-11-15 04:42:13,379] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:09,650] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:44:09,650] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:07,052] INFO : Step 32000/500000 rank0 | Train Loss: 2.0221 | Train Loss Unweighted: 0.7242 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2300 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2261 | Regression_eta: 0.0333 | Regression_pt: 0.2180 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0889 | Classification_binary: 0.2300 | Regression_cos_phi: 0.4763 | Regression_energy: 0.1353 | Regression_eta: 0.4320 | Regression_pt: 0.1304 | Regression_sin_phi: 0.5293 | LR: 7.19e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.6176s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0733 | max_loss: 0.0028 | max_backward: 0.1612 | max_optimizer_step: 0.0749 | max_tensorboard_logging: 1.0033 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7422.0 | uss_MB: 6276.5 | child_rss_MB: 23817.2 | allocated_MB: 198.6 | reserved_MB: 19218.0 | inactive_split_MB: 569.4 +rank0 [2026-11-15 04:46:07,054] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 04:46:07,127] INFO : Running validation on rank0 +rank0 [2026-11-15 04:46:07,128] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:46:07,128] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:46:07,749] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 04:46:07,870] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:46:07,985] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:46:08,057] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:46:08,125] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:46:08,190] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:46:08,257] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:46:08,323] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:46:08,430] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:46:08,533] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:46:08,635] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:46:08,702] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:46:08,768] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:46:08,833] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:46:08,898] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:46:08,963] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:46:09,066] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:46:09,091] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:46:09,104] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:46:09,107] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:46:09,117] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:46:09,130] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:46:09,133] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:46:09,143] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:46:09,156] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:46:09,172] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:46:09,174] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:46:09,175] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:46:09,176] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:46:09,177] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:46:09,178] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:46:09,179] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:46:16,870] INFO : VALIDATION | Step=32000/500000 | Train Loss=2.0221 | Valid Loss=2.7485 | Train Loss Unweighted=0.7242 | Valid Loss Unweighted=0.9045 | Stale=4 | ETA=2154.9m +rank0 [2026-11-15 04:46:16,870] INFO : split_configs=['1'] +rank0 [2026-11-15 04:46:16,912] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:46:16,921] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:46:17,010] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:46:17,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:18,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:18,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:18,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:19,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:19,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:19,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:20,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:20,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:20,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:21,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:21,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:21,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:22,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:22,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:22,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:23,023] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 04:46:23,026] INFO : split_configs=['1'] +rank0 [2026-11-15 04:46:23,062] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:46:23,065] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:46:23,126] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:46:24,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:24,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:25,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:25,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:25,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:26,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:26,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:27,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:27,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:28,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:28,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:29,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:29,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:29,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:30,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:30,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:30,901] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:46:30,904] INFO : split_configs=['1'] +rank0 [2026-11-15 04:46:30,958] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:46:30,961] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:46:31,027] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:46:32,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:32,465] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:32,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:33,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:33,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:34,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:34,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:34,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:35,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:35,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:36,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:36,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:36,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:37,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:37,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:38,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:38,273] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:46:38,280] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf +rank0 [2026-11-15 04:46:39,415] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_qq_pf +rank0 [2026-11-15 04:46:39,831] INFO : Plotted number of elements +rank0 [2026-11-15 04:46:41,116] INFO : Plotted jets +rank0 [2026-11-15 04:46:42,584] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:42,596] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:46:44,181] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:46:44,616] INFO : Plotted number of elements +rank0 [2026-11-15 04:46:45,939] INFO : Plotted jets +rank0 [2026-11-15 04:46:46,962] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:46,971] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:46:48,373] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_32000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:46:48,816] INFO : Plotted number of elements +rank0 [2026-11-15 04:46:50,122] INFO : Plotted jets +rank0 [2026-11-15 04:46:51,105] INFO : Plotted jet ratio +rank0 [2026-11-15 04:50:44,246] INFO : Step 33000/500000 rank0 | Train Loss: 2.7063 | Train Loss Unweighted: 0.9333 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2388 | Regression_cos_phi: 0.0118 | Regression_energy: 0.3127 | Regression_eta: 0.0551 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1030 | Classification_binary: 0.2388 | Regression_cos_phi: 0.6910 | Regression_energy: 0.1871 | Regression_eta: 0.7161 | Regression_pt: 0.1825 | Regression_sin_phi: 0.5878 | LR: 7.51e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.5544s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0024 | max_forward: 0.0588 | max_loss: 0.0027 | max_backward: 0.1248 | max_optimizer_step: 0.0881 | max_tensorboard_logging: 0.9698 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 7835.4 | uss_MB: 6689.9 | child_rss_MB: 23727.0 | allocated_MB: 195.7 | reserved_MB: 19098.0 | inactive_split_MB: 108.3 +rank0 [2026-11-15 04:50:44,251] INFO : Running validation on rank0 +rank0 [2026-11-15 04:50:44,251] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:50:44,251] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:50:44,882] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 04:50:45,002] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:50:45,118] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:50:45,188] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:50:45,255] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:50:45,321] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:50:45,388] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:50:45,455] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:50:45,566] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:50:45,670] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:50:45,772] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:50:45,841] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:50:45,909] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:50:45,976] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:50:46,045] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:50:46,114] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:50:46,218] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:50:46,225] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:50:46,240] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:50:46,241] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:50:46,243] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:50:46,244] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:50:46,245] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:50:46,252] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:50:46,254] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:50:46,255] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:50:46,256] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:50:46,257] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:50:46,258] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:50:46,259] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:50:46,260] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:50:46,261] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:50:54,032] INFO : VALIDATION | Step=33000/500000 | Train Loss=2.7063 | Valid Loss=2.6776 | Train Loss Unweighted=0.9333 | Valid Loss Unweighted=0.8917 | Stale=0 | ETA=2150.5m +rank0 [2026-11-15 04:50:54,033] INFO : split_configs=['1'] +rank0 [2026-11-15 04:50:54,074] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:50:54,082] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:50:54,177] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:50:55,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:50:55,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:50:55,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:50:56,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:50:56,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:50:56,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:50:57,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:50:57,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:50:57,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:50:58,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:50:58,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:50:58,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:50:59,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:50:59,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:50:59,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:00,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:00,241] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:51:00,244] INFO : split_configs=['1'] +rank0 [2026-11-15 04:51:00,285] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:51:00,289] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:51:00,351] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:01,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:01,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:02,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:02,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:03,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:03,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:04,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:04,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:04,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:05,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:05,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:06,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:06,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:07,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:07,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:08,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:08,142] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:51:08,145] INFO : split_configs=['1'] +rank0 [2026-11-15 04:51:08,185] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:51:08,189] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:51:08,253] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:09,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:09,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:10,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:10,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:10,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:11,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:11,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:12,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:12,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:13,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:13,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:13,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:14,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:14,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:15,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:15,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:15,602] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:51:15,609] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf +rank0 [2026-11-15 04:51:16,751] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_qq_pf +rank0 [2026-11-15 04:51:17,179] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:18,485] INFO : Plotted jets +rank0 [2026-11-15 04:51:19,974] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:19,986] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:51:21,569] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:51:22,004] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:23,356] INFO : Plotted jets +rank0 [2026-11-15 04:51:24,382] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:24,392] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:51:25,801] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_33000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:51:26,266] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:27,581] INFO : Plotted jets +rank0 [2026-11-15 04:51:28,581] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:29,008] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:51:29,008] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:55:22,934] INFO : Step 34000/500000 rank0 | Train Loss: 1.8600 | Train Loss Unweighted: 0.9171 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2606 | Regression_cos_phi: 0.0069 | Regression_energy: 0.3089 | Regression_eta: 0.0237 | Regression_pt: 0.3102 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1140 | Classification_binary: 0.2606 | Regression_cos_phi: 0.4077 | Regression_energy: 0.1848 | Regression_eta: 0.3083 | Regression_pt: 0.1855 | Regression_sin_phi: 0.3992 | LR: 7.83e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.4464s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0658 | max_loss: 0.0030 | max_backward: 0.1392 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7918.8 | uss_MB: 6773.3 | child_rss_MB: 23850.9 | allocated_MB: 199.5 | reserved_MB: 18038.0 | inactive_split_MB: 312.5 +rank0 [2026-11-15 04:55:22,937] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 04:55:23,005] INFO : Running validation on rank0 +rank0 [2026-11-15 04:55:23,005] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:55:23,005] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:55:23,626] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 04:55:23,746] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:55:23,858] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:55:23,927] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:55:23,994] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:55:24,061] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:55:24,129] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:55:24,197] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:55:24,307] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:55:24,413] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:55:24,519] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:55:24,590] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:55:24,658] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:55:24,726] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:55:24,793] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:55:24,859] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:55:24,962] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:55:24,969] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:55:24,972] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:55:24,974] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:55:24,977] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:55:24,979] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:55:24,982] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:55:24,984] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:55:24,987] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:55:24,989] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:55:24,991] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:55:24,994] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:55:24,997] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:55:25,040] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:55:25,044] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:55:25,046] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:55:32,801] INFO : VALIDATION | Step=34000/500000 | Train Loss=1.8600 | Valid Loss=2.6758 | Train Loss Unweighted=0.9171 | Valid Loss Unweighted=0.9024 | Stale=0 | ETA=2146.4m +rank0 [2026-11-15 04:55:32,802] INFO : split_configs=['1'] +rank0 [2026-11-15 04:55:32,813] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 04:55:32,833] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 04:55:32,917] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:33,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:34,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:34,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:34,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:35,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:35,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:35,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:36,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:36,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:36,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:37,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:37,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:37,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:38,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:38,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:38,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:38,948] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 04:55:38,951] INFO : split_configs=['1'] +rank0 [2026-11-15 04:55:38,963] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 04:55:38,966] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 04:55:39,025] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:40,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:40,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:40,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:41,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:41,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:42,236] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:42,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:43,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:43,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:44,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:44,456] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:44,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:45,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:45,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:46,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:46,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:46,731] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:55:46,734] INFO : split_configs=['1'] +rank0 [2026-11-15 04:55:46,745] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 04:55:46,749] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:55:46,809] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:47,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:48,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:48,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:49,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:49,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:49,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:50,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:50,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:51,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:51,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:51,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:52,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:52,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:53,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:53,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:53,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:54,064] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 04:55:54,072] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf +rank0 [2026-11-15 04:55:55,223] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_qq_pf +rank0 [2026-11-15 04:55:55,674] INFO : Plotted number of elements +rank0 [2026-11-15 04:55:56,964] INFO : Plotted jets +rank0 [2026-11-15 04:55:58,479] INFO : Plotted jet ratio +rank0 [2026-11-15 04:55:58,490] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:56:00,092] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ttbar_pf +rank0 [2026-11-15 04:56:00,545] INFO : Plotted number of elements +rank0 [2026-11-15 04:56:01,890] INFO : Plotted jets +rank0 [2026-11-15 04:56:02,927] INFO : Plotted jet ratio +rank0 [2026-11-15 04:56:02,938] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:56:04,365] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_34000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 04:56:04,817] INFO : Plotted number of elements +rank0 [2026-11-15 04:56:06,145] INFO : Plotted jets +rank0 [2026-11-15 04:56:07,145] INFO : Plotted jet ratio +rank0 [2026-11-15 04:58:03,514] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:58:03,514] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:00:00,551] INFO : Step 35000/500000 rank0 | Train Loss: 1.6872 | Train Loss Unweighted: 0.8917 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2617 | Regression_cos_phi: 0.0053 | Regression_energy: 0.2936 | Regression_eta: 0.0196 | Regression_pt: 0.3045 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2617 | Regression_cos_phi: 0.3132 | Regression_energy: 0.1757 | Regression_eta: 0.2545 | Regression_pt: 0.1821 | Regression_sin_phi: 0.4228 | LR: 8.16e-05 | DataLoad Time: 0.0023s | Model Forward Time: 1.5350s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0029 | max_forward: 0.0766 | max_loss: 0.0033 | max_backward: 0.1700 | max_optimizer_step: 0.0795 | max_tensorboard_logging: 0.8781 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7895.6 | uss_MB: 6750.1 | child_rss_MB: 23888.8 | allocated_MB: 198.8 | reserved_MB: 18350.0 | inactive_split_MB: 293.2 +rank0 [2026-11-15 05:00:00,553] INFO : Running validation on rank0 +rank0 [2026-11-15 05:00:00,553] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:00:00,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:00:01,175] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 05:00:01,297] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:00:01,408] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:00:01,478] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:00:01,544] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:00:01,610] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:00:01,677] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:00:01,745] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:00:01,855] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:00:01,962] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:00:02,068] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:00:02,137] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:00:02,204] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:00:02,270] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:00:02,336] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:00:02,402] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:00:02,504] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:00:02,512] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:00:02,515] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:00:02,517] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:00:02,520] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:00:02,522] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:00:02,525] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:00:02,527] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:00:02,530] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:00:02,532] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:00:02,534] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:00:02,537] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:00:02,539] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:00:02,542] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:00:02,546] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:00:02,548] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:00:10,277] INFO : VALIDATION | Step=35000/500000 | Train Loss=1.6872 | Valid Loss=2.6696 | Train Loss Unweighted=0.8917 | Valid Loss Unweighted=0.8862 | Stale=0 | ETA=2142.1m +rank0 [2026-11-15 05:00:10,277] INFO : split_configs=['1'] +rank0 [2026-11-15 05:00:10,288] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:00:10,296] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:00:10,387] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:00:11,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:00:11,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:00:11,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:00:12,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:00:12,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:00:13,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:00:13,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:00:13,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:00:14,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:00:14,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:00:14,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:00:15,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:00:15,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:00:15,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:00:15,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:00:16,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:00:16,445] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:00:16,448] INFO : split_configs=['1'] +rank0 [2026-11-15 05:00:16,460] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:00:16,464] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:00:16,528] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:00:17,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:00:18,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:00:18,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:00:18,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:00:19,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:00:19,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:00:20,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:00:20,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:00:21,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:00:21,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:00:22,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:00:22,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:00:22,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:00:23,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:00:23,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:00:24,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:00:24,366] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:00:24,369] INFO : split_configs=['1'] +rank0 [2026-11-15 05:00:24,381] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:00:24,384] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:00:24,449] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:00:25,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:00:25,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:00:26,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:00:26,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:00:27,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:00:27,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:00:27,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:00:28,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:00:28,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:00:29,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:00:29,615] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:00:30,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:00:30,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:00:30,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:00:31,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:00:31,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:00:31,768] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:00:31,788] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf +rank0 [2026-11-15 05:00:32,958] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_qq_pf +rank0 [2026-11-15 05:00:33,388] INFO : Plotted number of elements +rank0 [2026-11-15 05:00:34,684] INFO : Plotted jets +rank0 [2026-11-15 05:00:36,195] INFO : Plotted jet ratio +rank0 [2026-11-15 05:00:36,207] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:00:37,787] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:00:38,223] INFO : Plotted number of elements +rank0 [2026-11-15 05:00:39,569] INFO : Plotted jets +rank0 [2026-11-15 05:00:40,606] INFO : Plotted jet ratio +rank0 [2026-11-15 05:00:40,614] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:00:42,023] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_35000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:00:42,476] INFO : Plotted number of elements +rank0 [2026-11-15 05:00:43,787] INFO : Plotted jets +rank0 [2026-11-15 05:00:44,784] INFO : Plotted jet ratio +rank0 [2026-11-15 05:04:39,262] INFO : Step 36000/500000 rank0 | Train Loss: 1.6653 | Train Loss Unweighted: 0.8876 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2262 | Regression_cos_phi: 0.0049 | Regression_energy: 0.3084 | Regression_eta: 0.0170 | Regression_pt: 0.3235 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1020 | Classification_binary: 0.2262 | Regression_cos_phi: 0.2849 | Regression_energy: 0.1845 | Regression_eta: 0.2213 | Regression_pt: 0.1935 | Regression_sin_phi: 0.4529 | LR: 8.50e-05 | DataLoad Time: 0.0021s | Model Forward Time: 1.4649s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0592 | max_loss: 0.0033 | max_backward: 0.1277 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7969.3 | uss_MB: 6823.8 | child_rss_MB: 23809.6 | allocated_MB: 195.4 | reserved_MB: 20320.0 | inactive_split_MB: 186.6 +rank0 [2026-11-15 05:04:39,264] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-36000.pth +rank0 [2026-11-15 05:04:39,332] INFO : Running validation on rank0 +rank0 [2026-11-15 05:04:39,333] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:04:39,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:04:39,962] INFO : saved batch0_epoch36000.parquet +rank0 [2026-11-15 05:04:40,086] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:04:40,200] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:04:40,269] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:04:40,337] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:04:40,403] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:04:40,470] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:04:40,539] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:04:40,650] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:04:40,758] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:04:40,864] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:04:40,935] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:04:41,004] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:04:41,073] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:04:41,141] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:04:41,208] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:04:41,309] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:04:41,316] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:04:41,319] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:04:41,321] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:04:41,324] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:04:41,328] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:04:41,330] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:04:41,332] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:04:41,336] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:04:41,338] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:04:41,341] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:04:41,343] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:04:41,346] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:04:41,348] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:04:41,351] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:04:41,353] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:04:49,141] INFO : VALIDATION | Step=36000/500000 | Train Loss=1.6653 | Valid Loss=2.6418 | Train Loss Unweighted=0.8876 | Valid Loss Unweighted=0.8849 | Stale=0 | ETA=2138.0m +rank0 [2026-11-15 05:04:49,142] INFO : split_configs=['1'] +rank0 [2026-11-15 05:04:49,153] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:04:49,162] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:04:49,255] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:04:50,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:04:50,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:04:50,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:04:51,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:04:51,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:04:52,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:04:52,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:04:52,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:04:53,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:04:53,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:04:53,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:04:54,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:04:54,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:04:54,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:04:55,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:04:55,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:04:55,709] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:04:55,712] INFO : split_configs=['1'] +rank0 [2026-11-15 05:04:55,723] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:04:55,729] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:04:55,794] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:04:56,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:04:57,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:04:57,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:04:58,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:04:58,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:04:59,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:04:59,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:05:00,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:05:00,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:05:01,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:05:01,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:05:02,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:05:02,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:05:03,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:05:03,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:05:04,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:05:04,103] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:05:04,108] INFO : split_configs=['1'] +rank0 [2026-11-15 05:05:04,120] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:05:04,123] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:05:04,193] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:05:05,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:05:05,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:05:06,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:05:06,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:05:06,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:05:07,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:05:07,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:05:08,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:05:08,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:05:08,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:05:09,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:05:09,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:05:10,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:05:10,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:05:11,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:05:11,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:05:11,521] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:05:11,530] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf +rank0 [2026-11-15 05:05:12,693] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_qq_pf +rank0 [2026-11-15 05:05:13,121] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:14,412] INFO : Plotted jets +rank0 [2026-11-15 05:05:15,889] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:15,911] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:05:17,534] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:05:17,967] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:19,307] INFO : Plotted jets +rank0 [2026-11-15 05:05:20,325] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:20,334] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:05:21,758] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_36000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:05:22,216] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:23,518] INFO : Plotted jets +rank0 [2026-11-15 05:05:24,497] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:24,911] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 05:05:24,911] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:09:17,756] INFO : Step 37000/500000 rank0 | Train Loss: 2.6032 | Train Loss Unweighted: 0.9198 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2519 | Regression_cos_phi: 0.0118 | Regression_energy: 0.3004 | Regression_eta: 0.0590 | Regression_pt: 0.2894 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0978 | Classification_binary: 0.2519 | Regression_cos_phi: 0.6929 | Regression_energy: 0.1798 | Regression_eta: 0.7659 | Regression_pt: 0.1731 | Regression_sin_phi: 0.4418 | LR: 8.85e-05 | DataLoad Time: 0.0017s | Model Forward Time: 1.4264s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0561 | max_loss: 0.0029 | max_backward: 0.1173 | max_optimizer_step: 0.0801 | max_tensorboard_logging: 0.8205 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8132.9 | uss_MB: 6987.5 | child_rss_MB: 23797.0 | allocated_MB: 196.0 | reserved_MB: 21064.0 | inactive_split_MB: 400.0 +rank0 [2026-11-15 05:09:17,799] INFO : Running validation on rank0 +rank0 [2026-11-15 05:09:17,799] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:09:17,799] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:09:18,430] INFO : saved batch0_epoch37000.parquet +rank0 [2026-11-15 05:09:18,556] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:09:18,671] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:09:18,740] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:09:18,805] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:09:18,871] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:09:18,938] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:09:19,004] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:09:19,117] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:09:19,222] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:09:19,328] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:09:19,398] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:09:19,466] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:09:19,534] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:09:19,603] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:09:19,669] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:09:19,773] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:09:19,780] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:09:19,783] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:09:19,785] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:09:19,788] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:09:19,792] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:09:19,794] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:09:19,797] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:09:19,799] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:09:19,802] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:09:19,804] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:09:19,807] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:09:19,809] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:09:19,811] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:09:19,814] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:09:19,816] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:09:27,488] INFO : VALIDATION | Step=37000/500000 | Train Loss=2.6032 | Valid Loss=2.6474 | Train Loss Unweighted=0.9198 | Valid Loss Unweighted=0.8803 | Stale=1 | ETA=2133.8m +rank0 [2026-11-15 05:09:27,489] INFO : split_configs=['1'] +rank0 [2026-11-15 05:09:27,500] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:09:27,508] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:09:27,599] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:09:28,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:09:28,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:09:29,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:09:29,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:09:29,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:09:30,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:09:30,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:09:30,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:09:31,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:09:31,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:09:31,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:09:32,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:09:32,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:09:32,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:09:33,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:09:33,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:09:33,627] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:09:33,630] INFO : split_configs=['1'] +rank0 [2026-11-15 05:09:33,643] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:09:33,646] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:09:33,707] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:09:34,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:09:35,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:09:35,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:09:36,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:09:36,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:09:36,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:09:37,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:09:37,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:09:38,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:09:38,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:09:39,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:09:39,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:09:40,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:09:40,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:09:40,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:09:41,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:09:41,427] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:09:41,430] INFO : split_configs=['1'] +rank0 [2026-11-15 05:09:41,441] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:09:41,445] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:09:41,510] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:09:42,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:09:42,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:09:43,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:09:43,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:09:44,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:09:44,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:09:44,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:09:45,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:09:45,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:09:46,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:09:46,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:09:46,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:09:47,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:09:47,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:09:48,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:09:48,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:09:48,695] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 05:09:48,702] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf +rank0 [2026-11-15 05:09:49,854] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_qq_pf +rank0 [2026-11-15 05:09:50,289] INFO : Plotted number of elements +rank0 [2026-11-15 05:09:51,574] INFO : Plotted jets +rank0 [2026-11-15 05:09:53,086] INFO : Plotted jet ratio +rank0 [2026-11-15 05:09:53,098] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:09:54,738] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:09:55,188] INFO : Plotted number of elements +rank0 [2026-11-15 05:09:56,540] INFO : Plotted jets +rank0 [2026-11-15 05:09:57,570] INFO : Plotted jet ratio +rank0 [2026-11-15 05:09:57,579] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:09:59,091] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_37000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:09:59,538] INFO : Plotted number of elements +rank0 [2026-11-15 05:10:00,859] INFO : Plotted jets +rank0 [2026-11-15 05:10:01,846] INFO : Plotted jet ratio +rank0 [2026-11-15 05:11:58,720] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 05:11:58,720] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:13:55,956] INFO : Step 38000/500000 rank0 | Train Loss: 2.1653 | Train Loss Unweighted: 0.8310 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2411 | Regression_cos_phi: 0.0056 | Regression_energy: 0.2672 | Regression_eta: 0.0563 | Regression_pt: 0.2532 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1003 | Classification_binary: 0.2411 | Regression_cos_phi: 0.3311 | Regression_energy: 0.1599 | Regression_eta: 0.7318 | Regression_pt: 0.1515 | Regression_sin_phi: 0.4497 | LR: 9.21e-05 | DataLoad Time: 0.0022s | Model Forward Time: 1.4117s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0021 | max_forward: 0.0516 | max_loss: 0.0029 | max_backward: 0.1099 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.8435 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8083.8 | uss_MB: 6938.5 | child_rss_MB: 23850.3 | allocated_MB: 190.0 | reserved_MB: 20456.0 | inactive_split_MB: 218.0 +rank0 [2026-11-15 05:13:55,959] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-38000.pth +rank0 [2026-11-15 05:13:56,027] INFO : Running validation on rank0 +rank0 [2026-11-15 05:13:56,027] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:13:56,027] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:13:56,645] INFO : saved batch0_epoch38000.parquet +rank0 [2026-11-15 05:13:56,768] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:13:56,882] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:13:56,953] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:13:57,021] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:13:57,091] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:13:57,160] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:13:57,230] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:13:57,337] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:13:57,442] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:13:57,544] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:13:57,612] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:13:57,678] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:13:57,743] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:13:57,809] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:13:57,873] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:13:57,974] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:13:57,981] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:13:57,984] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:13:57,986] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:13:57,989] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:13:57,992] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:13:57,994] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:13:57,996] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:13:57,999] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:13:58,001] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:13:58,004] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:13:58,006] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:13:58,009] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:13:58,011] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:13:58,014] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:13:58,016] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:14:05,728] INFO : VALIDATION | Step=38000/500000 | Train Loss=2.1653 | Valid Loss=2.6244 | Train Loss Unweighted=0.8310 | Valid Loss Unweighted=0.8814 | Stale=0 | ETA=2129.5m +rank0 [2026-11-15 05:14:05,729] INFO : split_configs=['1'] +rank0 [2026-11-15 05:14:05,746] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:14:05,753] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:14:05,847] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:14:06,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:14:07,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:14:07,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:14:07,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:14:08,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:14:08,456] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:14:08,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:14:09,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:14:09,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:14:09,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:14:10,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:14:10,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:14:10,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:14:11,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:14:11,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:14:11,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:14:11,901] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:14:11,904] INFO : split_configs=['1'] +rank0 [2026-11-15 05:14:11,916] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:14:11,921] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:14:11,987] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:14:13,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:14:13,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:14:13,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:14:14,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:14:14,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:14:15,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:14:15,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:14:16,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:14:16,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:14:17,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:14:17,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:14:17,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:14:18,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:14:18,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:14:19,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:14:19,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:14:19,794] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:14:19,798] INFO : split_configs=['1'] +rank0 [2026-11-15 05:14:19,809] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:14:19,813] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:14:19,879] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:14:20,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:14:21,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:14:21,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:14:22,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:14:22,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:14:22,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:14:23,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:14:23,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:14:24,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:14:24,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:14:25,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:14:25,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:14:25,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:14:26,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:14:26,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:14:27,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:14:27,160] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:14:27,168] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf +rank0 [2026-11-15 05:14:28,328] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_qq_pf +rank0 [2026-11-15 05:14:28,752] INFO : Plotted number of elements +rank0 [2026-11-15 05:14:30,042] INFO : Plotted jets +rank0 [2026-11-15 05:14:31,544] INFO : Plotted jet ratio +rank0 [2026-11-15 05:14:31,556] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:14:33,151] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:14:33,594] INFO : Plotted number of elements +rank0 [2026-11-15 05:14:34,928] INFO : Plotted jets +rank0 [2026-11-15 05:14:35,961] INFO : Plotted jet ratio +rank0 [2026-11-15 05:14:35,970] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:14:37,406] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_38000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:14:37,859] INFO : Plotted number of elements +rank0 [2026-11-15 05:14:39,178] INFO : Plotted jets +rank0 [2026-11-15 05:14:40,166] INFO : Plotted jet ratio +rank0 [2026-11-15 05:18:33,321] INFO : Step 39000/500000 rank0 | Train Loss: 2.1727 | Train Loss Unweighted: 0.9065 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2366 | Regression_cos_phi: 0.0088 | Regression_energy: 0.3197 | Regression_eta: 0.0391 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1076 | Classification_binary: 0.2366 | Regression_cos_phi: 0.5173 | Regression_energy: 0.1913 | Regression_eta: 0.5076 | Regression_pt: 0.1764 | Regression_sin_phi: 0.4359 | LR: 9.57e-05 | DataLoad Time: 0.0148s | Model Forward Time: 1.4401s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0028 | max_forward: 0.0602 | max_loss: 0.0029 | max_backward: 0.1260 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.8488 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8190.6 | uss_MB: 7045.2 | child_rss_MB: 23908.5 | allocated_MB: 198.3 | reserved_MB: 22482.0 | inactive_split_MB: 383.7 +rank0 [2026-11-15 05:18:33,323] INFO : Running validation on rank0 +rank0 [2026-11-15 05:18:33,324] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:18:33,324] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:18:33,948] INFO : saved batch0_epoch39000.parquet +rank0 [2026-11-15 05:18:34,070] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:18:34,185] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:18:34,255] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:18:34,323] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:18:34,389] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:18:34,457] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:18:34,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:18:34,636] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:18:34,742] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:18:34,845] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:18:34,912] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:18:34,978] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:18:35,045] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:18:35,112] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:18:35,179] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:18:35,281] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:18:35,288] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:18:35,291] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:18:35,293] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:18:35,296] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:18:35,299] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:18:35,301] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:18:35,303] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:18:35,306] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:18:35,308] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:18:35,311] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:18:35,313] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:18:35,316] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:18:35,318] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:18:35,321] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:18:35,323] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:18:43,071] INFO : VALIDATION | Step=39000/500000 | Train Loss=2.1727 | Valid Loss=2.5693 | Train Loss Unweighted=0.9065 | Valid Loss Unweighted=0.8747 | Stale=0 | ETA=2125.1m +rank0 [2026-11-15 05:18:43,071] INFO : split_configs=['1'] +rank0 [2026-11-15 05:18:43,082] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:18:43,090] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:18:43,189] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:18:44,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:18:44,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:18:44,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:18:45,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:18:45,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:18:45,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:18:46,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:18:46,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:18:46,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:18:47,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:18:47,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:18:47,890] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:18:48,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:18:48,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:18:48,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:18:49,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:18:49,246] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:18:49,248] INFO : split_configs=['1'] +rank0 [2026-11-15 05:18:49,259] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:18:49,263] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:18:49,328] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:18:50,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:18:50,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:18:51,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:18:51,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:18:52,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:18:52,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:18:53,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:18:53,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:18:53,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:18:54,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:18:54,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:18:55,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:18:55,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:18:56,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:18:56,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:18:57,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:18:57,157] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:18:57,160] INFO : split_configs=['1'] +rank0 [2026-11-15 05:18:57,172] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:18:57,175] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:18:57,243] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:18:58,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:18:58,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:18:59,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:18:59,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:18:59,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:19:00,382] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:19:00,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:19:01,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:19:01,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:19:01,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:19:02,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:19:02,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:19:03,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:19:03,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:19:04,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:19:04,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:19:04,548] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:19:04,554] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf +rank0 [2026-11-15 05:19:05,729] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_qq_pf +rank0 [2026-11-15 05:19:06,153] INFO : Plotted number of elements +rank0 [2026-11-15 05:19:07,445] INFO : Plotted jets +rank0 [2026-11-15 05:19:08,931] INFO : Plotted jet ratio +rank0 [2026-11-15 05:19:08,942] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:19:10,614] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:19:11,057] INFO : Plotted number of elements +rank0 [2026-11-15 05:19:12,391] INFO : Plotted jets +rank0 [2026-11-15 05:19:13,420] INFO : Plotted jet ratio +rank0 [2026-11-15 05:19:13,428] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:19:14,943] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_39000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:19:15,394] INFO : Plotted number of elements +rank0 [2026-11-15 05:19:16,704] INFO : Plotted jets +rank0 [2026-11-15 05:19:17,693] INFO : Plotted jet ratio +rank0 [2026-11-15 05:19:18,135] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 05:19:18,135] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:23:11,791] INFO : Step 40000/500000 rank0 | Train Loss: 2.6374 | Train Loss Unweighted: 0.8317 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2701 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2619 | Regression_eta: 0.0467 | Regression_pt: 0.2304 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1128 | Classification_binary: 0.2701 | Regression_cos_phi: 0.6592 | Regression_energy: 0.1568 | Regression_eta: 0.6061 | Regression_pt: 0.1378 | Regression_sin_phi: 0.6947 | LR: 9.94e-05 | DataLoad Time: 0.0023s | Model Forward Time: 1.5348s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0677 | max_loss: 0.0029 | max_backward: 0.1446 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.8439 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8259.3 | uss_MB: 7114.0 | child_rss_MB: 23641.0 | allocated_MB: 201.2 | reserved_MB: 16446.0 | inactive_split_MB: 560.8 +rank0 [2026-11-15 05:23:11,796] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-40000.pth +rank0 [2026-11-15 05:23:11,866] INFO : Running validation on rank0 +rank0 [2026-11-15 05:23:11,866] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:23:11,866] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:23:12,495] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-15 05:23:12,623] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:23:12,736] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:23:12,808] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:23:12,875] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:23:12,941] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:23:13,007] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:23:13,077] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:23:13,189] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:23:13,297] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:23:13,403] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:23:13,474] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:23:13,542] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:23:13,608] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:23:13,674] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:23:13,741] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:23:13,844] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:23:13,852] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:23:13,854] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:23:13,857] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:23:13,859] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:23:13,862] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:23:13,865] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:23:13,867] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:23:13,870] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:23:13,872] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:23:13,874] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:23:13,877] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:23:13,879] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:23:13,882] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:23:13,885] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:23:13,887] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:23:21,580] INFO : VALIDATION | Step=40000/500000 | Train Loss=2.6374 | Valid Loss=2.5945 | Train Loss Unweighted=0.8317 | Valid Loss Unweighted=0.8703 | Stale=1 | ETA=2120.8m +rank0 [2026-11-15 05:23:21,581] INFO : split_configs=['1'] +rank0 [2026-11-15 05:23:21,591] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:23:21,599] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:23:21,680] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:23:22,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:23:22,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:23:23,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:23:23,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:23:23,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:23:24,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:23:24,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:23:24,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:23:25,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:23:25,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:23:25,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:23:26,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:23:26,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:23:26,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:23:27,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:23:27,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:23:27,655] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 05:23:27,659] INFO : split_configs=['1'] +rank0 [2026-11-15 05:23:27,671] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:23:27,674] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:23:27,733] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:23:28,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:23:29,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:23:29,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:23:30,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:23:30,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:23:30,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:23:31,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:23:31,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:23:32,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:23:32,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:23:33,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:23:33,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:23:34,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:23:34,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:23:34,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:23:35,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:23:35,521] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:23:35,524] INFO : split_configs=['1'] +rank0 [2026-11-15 05:23:35,536] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:23:35,540] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:23:35,600] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:23:36,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:23:37,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:23:37,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:23:37,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:23:38,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:23:38,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:23:39,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:23:39,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:23:39,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:23:40,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:23:40,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:23:41,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:23:41,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:23:41,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:23:42,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:23:42,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:23:42,856] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:23:42,863] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf +rank0 [2026-11-15 05:23:44,035] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_qq_pf +rank0 [2026-11-15 05:23:44,466] INFO : Plotted number of elements +rank0 [2026-11-15 05:23:45,741] INFO : Plotted jets +rank0 [2026-11-15 05:23:47,236] INFO : Plotted jet ratio +rank0 [2026-11-15 05:23:47,247] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:23:48,882] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:23:49,333] INFO : Plotted number of elements +rank0 [2026-11-15 05:23:50,667] INFO : Plotted jets +rank0 [2026-11-15 05:23:51,695] INFO : Plotted jet ratio +rank0 [2026-11-15 05:23:51,703] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:23:53,145] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_40000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:23:53,592] INFO : Plotted number of elements +rank0 [2026-11-15 05:23:54,904] INFO : Plotted jets +rank0 [2026-11-15 05:23:55,898] INFO : Plotted jet ratio +rank0 [2026-11-15 05:25:52,071] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 05:25:52,072] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:27:49,452] INFO : Step 41000/500000 rank0 | Train Loss: 2.3469 | Train Loss Unweighted: 0.7935 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2603 | Regression_eta: 0.0263 | Regression_pt: 0.2458 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1359 | Classification_binary: 0.2387 | Regression_cos_phi: 0.6683 | Regression_energy: 0.1558 | Regression_eta: 0.3420 | Regression_pt: 0.1470 | Regression_sin_phi: 0.6593 | LR: 1.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5137s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0645 | max_loss: 0.0031 | max_backward: 0.1359 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.8821 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8438.9 | uss_MB: 7293.5 | child_rss_MB: 23753.6 | allocated_MB: 199.6 | reserved_MB: 21418.0 | inactive_split_MB: 220.4 +rank0 [2026-11-15 05:27:49,454] INFO : Running validation on rank0 +rank0 [2026-11-15 05:27:49,454] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:27:49,454] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:27:50,085] INFO : saved batch0_epoch41000.parquet +rank0 [2026-11-15 05:27:50,209] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:27:50,326] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:27:50,398] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:27:50,466] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:27:50,535] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:27:50,606] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:27:50,676] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:27:50,789] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:27:50,896] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:27:51,001] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:27:51,073] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:27:51,141] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:27:51,207] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:27:51,274] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:27:51,341] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:27:51,445] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:27:51,453] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:27:51,456] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:27:51,458] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:27:51,461] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:27:51,463] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:27:51,466] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:27:51,468] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:27:51,471] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:27:51,473] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:27:51,476] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:27:51,479] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:27:51,481] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:27:51,484] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:27:51,486] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:27:51,489] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:27:59,232] INFO : VALIDATION | Step=41000/500000 | Train Loss=2.3469 | Valid Loss=2.6093 | Train Loss Unweighted=0.7935 | Valid Loss Unweighted=0.8628 | Stale=2 | ETA=2116.4m +rank0 [2026-11-15 05:27:59,232] INFO : split_configs=['1'] +rank0 [2026-11-15 05:27:59,243] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:27:59,251] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:27:59,353] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:28:00,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:28:00,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:28:01,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:28:01,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:28:01,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:28:02,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:28:02,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:28:02,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:28:03,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:28:03,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:28:03,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:28:04,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:28:04,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:28:04,763] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:28:05,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:28:05,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:28:05,507] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:28:05,510] INFO : split_configs=['1'] +rank0 [2026-11-15 05:28:05,521] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:28:05,524] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:28:05,591] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:28:06,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:28:07,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:28:07,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:28:08,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:28:08,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:28:08,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:28:09,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:28:09,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:28:10,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:28:10,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:28:11,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:28:11,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:28:12,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:28:12,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:28:12,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:28:13,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:28:13,454] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:28:13,458] INFO : split_configs=['1'] +rank0 [2026-11-15 05:28:13,469] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:28:13,472] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:28:13,542] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:28:14,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:28:15,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:28:15,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:28:15,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:28:16,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:28:16,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:28:17,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:28:17,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:28:17,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:28:18,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:28:18,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:28:19,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:28:19,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:28:19,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:28:20,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:28:20,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:28:20,816] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 05:28:20,824] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf +rank0 [2026-11-15 05:28:22,002] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_qq_pf +rank0 [2026-11-15 05:28:22,433] INFO : Plotted number of elements +rank0 [2026-11-15 05:28:23,706] INFO : Plotted jets +rank0 [2026-11-15 05:28:25,214] INFO : Plotted jet ratio +rank0 [2026-11-15 05:28:25,226] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:28:26,866] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:28:27,297] INFO : Plotted number of elements +rank0 [2026-11-15 05:28:28,659] INFO : Plotted jets +rank0 [2026-11-15 05:28:29,690] INFO : Plotted jet ratio +rank0 [2026-11-15 05:28:29,699] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:28:31,175] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_41000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:28:31,609] INFO : Plotted number of elements +rank0 [2026-11-15 05:28:32,930] INFO : Plotted jets +rank0 [2026-11-15 05:28:33,906] INFO : Plotted jet ratio +rank0 [2026-11-15 05:32:27,411] INFO : Step 42000/500000 rank0 | Train Loss: 2.2835 | Train Loss Unweighted: 0.8997 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2219 | Regression_cos_phi: 0.0112 | Regression_energy: 0.3066 | Regression_eta: 0.0434 | Regression_pt: 0.3102 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0912 | Classification_binary: 0.2219 | Regression_cos_phi: 0.6580 | Regression_energy: 0.1835 | Regression_eta: 0.5637 | Regression_pt: 0.1855 | Regression_sin_phi: 0.3797 | LR: 1.07e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4428s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0636 | max_loss: 0.0028 | max_backward: 0.1360 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.8302 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8444.7 | uss_MB: 7299.3 | child_rss_MB: 23724.6 | allocated_MB: 195.0 | reserved_MB: 17728.0 | inactive_split_MB: 169.0 +rank0 [2026-11-15 05:32:27,413] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-42000.pth +rank0 [2026-11-15 05:32:27,489] INFO : Running validation on rank0 +rank0 [2026-11-15 05:32:27,490] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:32:27,490] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:32:28,114] INFO : saved batch0_epoch42000.parquet +rank0 [2026-11-15 05:32:28,237] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:32:28,353] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:32:28,425] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:32:28,493] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:32:28,561] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:32:28,632] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:32:28,700] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:32:28,811] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:32:28,915] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:32:29,017] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:32:29,089] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:32:29,156] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:32:29,221] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:32:29,288] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:32:29,353] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:32:29,456] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:32:29,469] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:32:29,472] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:32:29,474] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:32:29,477] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:32:29,480] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:32:29,482] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:32:29,484] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:32:29,487] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:32:29,489] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:32:29,492] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:32:29,495] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:32:29,497] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:32:29,500] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:32:29,503] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:32:29,505] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:32:37,181] INFO : VALIDATION | Step=42000/500000 | Train Loss=2.2835 | Valid Loss=2.5962 | Train Loss Unweighted=0.8997 | Valid Loss Unweighted=0.8641 | Stale=3 | ETA=2112.0m +rank0 [2026-11-15 05:32:37,181] INFO : split_configs=['1'] +rank0 [2026-11-15 05:32:37,192] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:32:37,199] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:32:37,291] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:32:38,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:32:38,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:32:38,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:32:39,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:32:39,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:32:39,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:32:40,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:32:40,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:32:40,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:32:41,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:32:41,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:32:42,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:32:42,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:32:42,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:32:42,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:32:43,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:32:43,415] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:32:43,418] INFO : split_configs=['1'] +rank0 [2026-11-15 05:32:43,430] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:32:43,434] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:32:43,500] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:32:44,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:32:45,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:32:45,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:32:45,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:32:46,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:32:46,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:32:47,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:32:47,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:32:48,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:32:48,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:32:49,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:32:49,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:32:49,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:32:50,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:32:50,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:32:51,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:32:51,375] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:32:51,378] INFO : split_configs=['1'] +rank0 [2026-11-15 05:32:51,389] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:32:51,392] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:32:51,463] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:32:52,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:32:53,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:32:53,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:32:53,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:32:54,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:32:54,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:32:55,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:32:55,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:32:55,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:32:56,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:32:56,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:32:57,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:32:57,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:32:57,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:32:58,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:32:58,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:32:58,850] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:32:58,857] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf +rank0 [2026-11-15 05:33:00,030] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_qq_pf +rank0 [2026-11-15 05:33:00,460] INFO : Plotted number of elements +rank0 [2026-11-15 05:33:01,772] INFO : Plotted jets +rank0 [2026-11-15 05:33:03,306] INFO : Plotted jet ratio +rank0 [2026-11-15 05:33:03,316] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:33:05,022] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:33:05,458] INFO : Plotted number of elements +rank0 [2026-11-15 05:33:06,806] INFO : Plotted jets +rank0 [2026-11-15 05:33:07,842] INFO : Plotted jet ratio +rank0 [2026-11-15 05:33:07,852] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:33:09,301] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_42000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:33:09,753] INFO : Plotted number of elements +rank0 [2026-11-15 05:33:11,078] INFO : Plotted jets +rank0 [2026-11-15 05:33:12,078] INFO : Plotted jet ratio +rank0 [2026-11-15 05:33:12,524] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 05:33:12,524] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:37:06,785] INFO : Step 43000/500000 rank0 | Train Loss: 2.4192 | Train Loss Unweighted: 0.9571 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2364 | Regression_cos_phi: 0.0123 | Regression_energy: 0.3280 | Regression_eta: 0.0325 | Regression_pt: 0.3391 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1114 | Classification_binary: 0.2364 | Regression_cos_phi: 0.7247 | Regression_energy: 0.1963 | Regression_eta: 0.4221 | Regression_pt: 0.2028 | Regression_sin_phi: 0.5255 | LR: 1.11e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4804s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0025 | max_forward: 0.0559 | max_loss: 0.0033 | max_backward: 0.1164 | max_optimizer_step: 0.0836 | max_tensorboard_logging: 0.8846 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6896.9 | uss_MB: 5751.6 | child_rss_MB: 23657.1 | allocated_MB: 195.4 | reserved_MB: 21536.0 | inactive_split_MB: 178.6 +rank0 [2026-11-15 05:37:06,787] INFO : Running validation on rank0 +rank0 [2026-11-15 05:37:06,788] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:37:06,788] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:37:07,419] INFO : saved batch0_epoch43000.parquet +rank0 [2026-11-15 05:37:07,548] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:37:07,662] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:37:07,733] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:37:07,800] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:37:07,868] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:37:07,936] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:37:08,004] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:37:08,118] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:37:08,225] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:37:08,331] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:37:08,401] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:37:08,468] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:37:08,533] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:37:08,600] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:37:08,667] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:37:08,771] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:37:08,778] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:37:08,779] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:37:08,783] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:37:08,784] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:37:08,788] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:37:08,789] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:37:08,790] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:37:08,792] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:37:08,793] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:37:08,793] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:37:08,795] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:37:08,796] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:37:08,797] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:37:08,798] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:37:08,813] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:37:16,529] INFO : VALIDATION | Step=43000/500000 | Train Loss=2.4192 | Valid Loss=2.6712 | Train Loss Unweighted=0.9571 | Valid Loss Unweighted=0.8719 | Stale=4 | ETA=2107.9m +rank0 [2026-11-15 05:37:16,529] INFO : split_configs=['1'] +rank0 [2026-11-15 05:37:16,540] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:37:16,548] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:37:16,638] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:37:17,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:37:17,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:37:18,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:37:18,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:37:18,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:37:19,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:37:19,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:37:19,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:37:20,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:37:20,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:37:20,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:37:21,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:37:21,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:37:21,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:37:22,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:37:22,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:37:22,711] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:37:22,714] INFO : split_configs=['1'] +rank0 [2026-11-15 05:37:22,726] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:37:22,735] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:37:22,798] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:37:23,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:37:24,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:37:24,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:37:25,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:37:25,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:37:26,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:37:26,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:37:26,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:37:27,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:37:27,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:37:28,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:37:28,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:37:29,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:37:29,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:37:30,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:37:30,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:37:30,615] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:37:30,618] INFO : split_configs=['1'] +rank0 [2026-11-15 05:37:30,629] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:37:30,632] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:37:30,695] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:37:31,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:37:32,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:37:32,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:37:32,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:37:33,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:37:33,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:37:34,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:37:34,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:37:34,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:37:35,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:37:35,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:37:36,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:37:36,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:37:37,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:37:37,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:37:37,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:37:37,945] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:37:37,952] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf +rank0 [2026-11-15 05:37:39,141] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_qq_pf +rank0 [2026-11-15 05:37:39,586] INFO : Plotted number of elements +rank0 [2026-11-15 05:37:40,887] INFO : Plotted jets +rank0 [2026-11-15 05:37:42,424] INFO : Plotted jet ratio +rank0 [2026-11-15 05:37:42,436] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:37:44,089] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:37:44,523] INFO : Plotted number of elements +rank0 [2026-11-15 05:37:45,874] INFO : Plotted jets +rank0 [2026-11-15 05:37:46,901] INFO : Plotted jet ratio +rank0 [2026-11-15 05:37:46,910] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:37:48,337] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_43000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:37:48,797] INFO : Plotted number of elements +rank0 [2026-11-15 05:37:50,118] INFO : Plotted jets +rank0 [2026-11-15 05:37:51,107] INFO : Plotted jet ratio +rank0 [2026-11-15 05:39:46,875] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 05:39:46,875] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:41:43,798] INFO : Step 44000/500000 rank0 | Train Loss: 2.5401 | Train Loss Unweighted: 0.9737 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2272 | Regression_cos_phi: 0.0074 | Regression_energy: 0.3460 | Regression_eta: 0.0824 | Regression_pt: 0.3052 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0906 | Classification_binary: 0.2272 | Regression_cos_phi: 0.4374 | Regression_energy: 0.2070 | Regression_eta: 1.0707 | Regression_pt: 0.1825 | Regression_sin_phi: 0.3244 | LR: 1.15e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4402s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0561 | max_loss: 0.0028 | max_backward: 0.1178 | max_optimizer_step: 0.0824 | max_tensorboard_logging: 0.8377 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8643.8 | uss_MB: 7498.5 | child_rss_MB: 23772.2 | allocated_MB: 194.6 | reserved_MB: 17098.0 | inactive_split_MB: 273.4 +rank0 [2026-11-15 05:41:43,800] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-44000.pth +rank0 [2026-11-15 05:41:43,867] INFO : Running validation on rank0 +rank0 [2026-11-15 05:41:43,867] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:41:43,867] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:41:44,483] INFO : saved batch0_epoch44000.parquet +rank0 [2026-11-15 05:41:44,603] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:41:44,719] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:41:44,790] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:41:44,857] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:41:44,922] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:41:44,989] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:41:45,056] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:41:45,166] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:41:45,270] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:41:45,372] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:41:45,439] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:41:45,504] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:41:45,569] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:41:45,634] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:41:45,700] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:41:45,802] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:41:45,809] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:41:45,812] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:41:45,815] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:41:45,817] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:41:45,820] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:41:45,822] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:41:45,825] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:41:45,828] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:41:45,830] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:41:45,832] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:41:45,836] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:41:45,838] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:41:45,841] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:41:45,844] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:41:45,846] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:41:53,545] INFO : VALIDATION | Step=44000/500000 | Train Loss=2.5401 | Valid Loss=2.6206 | Train Loss Unweighted=0.9737 | Valid Loss Unweighted=0.8645 | Stale=5 | ETA=2103.3m +rank0 [2026-11-15 05:41:53,545] INFO : split_configs=['1'] +rank0 [2026-11-15 05:41:53,556] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:41:53,564] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:41:53,658] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:41:54,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:41:54,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:41:55,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:41:55,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:41:56,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:41:56,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:41:56,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:41:57,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:41:57,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:41:57,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:41:58,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:41:58,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:41:58,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:41:59,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:41:59,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:41:59,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:41:59,823] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:41:59,825] INFO : split_configs=['1'] +rank0 [2026-11-15 05:41:59,837] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:41:59,841] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:41:59,911] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:42:01,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:42:01,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:42:01,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:42:02,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:42:02,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:42:03,217] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:42:03,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:42:04,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:42:04,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:42:05,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:42:05,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:42:05,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:42:06,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:42:06,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:42:07,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:42:07,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:42:07,800] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:42:07,803] INFO : split_configs=['1'] +rank0 [2026-11-15 05:42:07,814] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:42:07,819] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:42:07,891] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:42:08,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:42:09,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:42:09,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:42:10,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:42:10,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:42:11,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:42:11,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:42:11,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:42:12,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:42:12,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:42:13,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:42:13,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:42:13,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:42:14,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:42:14,759] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:42:15,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:42:15,275] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:42:15,282] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf +rank0 [2026-11-15 05:42:16,441] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_qq_pf +rank0 [2026-11-15 05:42:16,870] INFO : Plotted number of elements +rank0 [2026-11-15 05:42:18,167] INFO : Plotted jets +rank0 [2026-11-15 05:42:19,701] INFO : Plotted jet ratio +rank0 [2026-11-15 05:42:19,713] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:42:21,347] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:42:21,782] INFO : Plotted number of elements +rank0 [2026-11-15 05:42:23,144] INFO : Plotted jets +rank0 [2026-11-15 05:42:24,176] INFO : Plotted jet ratio +rank0 [2026-11-15 05:42:24,185] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:42:25,631] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_44000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:42:26,089] INFO : Plotted number of elements +rank0 [2026-11-15 05:42:27,416] INFO : Plotted jets +rank0 [2026-11-15 05:42:28,407] INFO : Plotted jet ratio +rank0 [2026-11-15 05:46:22,955] INFO : Step 45000/500000 rank0 | Train Loss: 2.0649 | Train Loss Unweighted: 0.7352 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2323 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2356 | Regression_eta: 0.0260 | Regression_pt: 0.2228 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1195 | Classification_binary: 0.2323 | Regression_cos_phi: 0.4968 | Regression_energy: 0.1410 | Regression_eta: 0.3377 | Regression_pt: 0.1332 | Regression_sin_phi: 0.6043 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4756s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0660 | max_loss: 0.0033 | max_backward: 0.1388 | max_optimizer_step: 0.0839 | max_tensorboard_logging: 0.8424 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 8646.7 | uss_MB: 7501.4 | child_rss_MB: 23760.3 | allocated_MB: 198.2 | reserved_MB: 22054.0 | inactive_split_MB: 161.8 +rank0 [2026-11-15 05:46:22,958] INFO : Running validation on rank0 +rank0 [2026-11-15 05:46:22,958] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:46:22,958] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:46:23,586] INFO : saved batch0_epoch45000.parquet +rank0 [2026-11-15 05:46:23,708] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:46:23,823] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:46:23,894] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:46:23,961] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:46:24,029] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:46:24,101] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:46:24,171] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:46:24,283] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:46:24,391] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:46:24,496] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:46:24,564] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:46:24,631] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:46:24,697] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:46:24,764] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:46:24,831] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:46:24,932] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:46:24,939] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:46:24,942] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:46:24,945] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:46:24,947] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:46:24,950] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:46:24,952] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:46:24,955] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:46:24,957] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:46:24,959] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:46:24,962] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:46:24,965] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:46:24,967] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:46:24,969] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:46:24,972] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:46:24,974] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:46:32,703] INFO : VALIDATION | Step=45000/500000 | Train Loss=2.0649 | Valid Loss=2.6115 | Train Loss Unweighted=0.7352 | Valid Loss Unweighted=0.8602 | Stale=6 | ETA=2099.1m +rank0 [2026-11-15 05:46:32,703] INFO : split_configs=['1'] +rank0 [2026-11-15 05:46:32,714] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:46:32,972] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:46:33,077] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:46:34,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:46:34,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:46:34,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:46:35,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:46:35,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:46:35,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:46:36,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:46:36,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:46:36,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:46:37,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:46:37,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:46:37,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:46:38,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:46:38,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:46:38,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:46:39,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:46:39,212] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:46:39,216] INFO : split_configs=['1'] +rank0 [2026-11-15 05:46:39,228] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:46:39,233] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:46:39,302] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:46:40,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:46:40,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:46:41,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:46:41,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:46:42,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:46:42,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:46:43,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:46:43,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:46:44,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:46:44,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:46:44,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:46:45,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:46:45,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:46:46,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:46:46,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:46:47,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:46:47,215] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:46:47,219] INFO : split_configs=['1'] +rank0 [2026-11-15 05:46:47,230] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:46:47,233] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:46:47,307] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:46:48,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:46:48,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:46:49,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:46:49,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:46:50,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:46:50,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:46:50,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:46:51,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:46:51,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:46:52,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:46:52,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:46:52,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:46:53,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:46:53,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:46:54,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:46:54,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:46:54,676] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:46:54,684] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf +rank0 [2026-11-15 05:46:55,861] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_qq_pf +rank0 [2026-11-15 05:46:56,286] INFO : Plotted number of elements +rank0 [2026-11-15 05:46:57,564] INFO : Plotted jets +rank0 [2026-11-15 05:46:59,093] INFO : Plotted jet ratio +rank0 [2026-11-15 05:46:59,104] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:47:00,701] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:47:01,134] INFO : Plotted number of elements +rank0 [2026-11-15 05:47:02,489] INFO : Plotted jets +rank0 [2026-11-15 05:47:03,534] INFO : Plotted jet ratio +rank0 [2026-11-15 05:47:03,543] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:47:04,965] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_45000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:47:05,426] INFO : Plotted number of elements +rank0 [2026-11-15 05:47:06,741] INFO : Plotted jets +rank0 [2026-11-15 05:47:07,724] INFO : Plotted jet ratio +rank0 [2026-11-15 05:47:08,185] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 05:47:08,185] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:51:01,376] INFO : Step 46000/500000 rank0 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9634 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2460 | Regression_cos_phi: 0.0060 | Regression_energy: 0.3387 | Regression_eta: 0.0353 | Regression_pt: 0.3326 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1010 | Classification_binary: 0.2460 | Regression_cos_phi: 0.3511 | Regression_energy: 0.2027 | Regression_eta: 0.4588 | Regression_pt: 0.1989 | Regression_sin_phi: 0.2779 | LR: 1.23e-04 | DataLoad Time: 0.0181s | Model Forward Time: 1.4919s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0560 | max_loss: 0.0031 | max_backward: 0.1182 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.8906 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 7136.7 | uss_MB: 5991.4 | child_rss_MB: 23800.4 | allocated_MB: 198.5 | reserved_MB: 19086.0 | inactive_split_MB: 161.5 +rank0 [2026-11-15 05:51:01,379] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-46000.pth +rank0 [2026-11-15 05:51:01,444] INFO : Running validation on rank0 +rank0 [2026-11-15 05:51:01,444] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:51:01,444] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:51:02,069] INFO : saved batch0_epoch46000.parquet +rank0 [2026-11-15 05:51:02,196] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:51:02,309] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:51:02,381] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:51:02,450] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:51:02,518] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:51:02,587] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:51:02,657] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:51:02,770] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:51:02,874] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:51:02,978] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:51:03,048] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:51:03,117] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:51:03,184] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:51:03,250] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:51:03,317] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:51:03,419] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:51:03,427] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:51:03,429] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:51:03,432] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:51:03,434] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:51:03,437] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:51:03,440] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:51:03,442] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:51:03,445] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:51:03,447] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:51:03,449] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:51:03,452] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:51:03,454] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:51:03,457] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:51:03,460] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:51:03,462] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:51:11,133] INFO : VALIDATION | Step=46000/500000 | Train Loss=1.8364 | Valid Loss=2.6188 | Train Loss Unweighted=0.9634 | Valid Loss Unweighted=0.8559 | Stale=7 | ETA=2094.8m +rank0 [2026-11-15 05:51:11,133] INFO : split_configs=['1'] +rank0 [2026-11-15 05:51:11,144] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:51:11,150] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:51:11,240] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:51:12,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:51:12,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:51:12,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:51:13,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:51:13,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:51:13,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:51:14,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:51:14,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:51:14,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:51:15,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:51:15,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:51:15,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:51:16,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:51:16,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:51:16,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:51:17,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:51:17,272] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:51:17,275] INFO : split_configs=['1'] +rank0 [2026-11-15 05:51:17,286] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:51:17,290] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:51:17,355] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:51:18,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:51:18,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:51:19,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:51:19,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:51:20,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:51:20,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:51:21,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:51:21,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:51:21,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:51:22,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:51:22,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:51:23,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:51:23,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:51:24,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:51:24,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:51:25,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:51:25,171] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:51:25,175] INFO : split_configs=['1'] +rank0 [2026-11-15 05:51:25,185] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:51:25,189] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:51:25,256] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:51:26,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:51:26,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:51:27,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:51:27,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:51:27,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:51:28,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:51:28,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:51:29,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:51:29,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:51:29,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:51:30,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:51:30,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:51:31,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:51:31,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:51:31,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:51:32,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:51:32,479] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 05:51:32,491] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf +rank0 [2026-11-15 05:51:33,667] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_qq_pf +rank0 [2026-11-15 05:51:34,111] INFO : Plotted number of elements +rank0 [2026-11-15 05:51:35,404] INFO : Plotted jets +rank0 [2026-11-15 05:51:36,933] INFO : Plotted jet ratio +rank0 [2026-11-15 05:51:36,944] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:51:38,591] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:51:39,025] INFO : Plotted number of elements +rank0 [2026-11-15 05:51:40,372] INFO : Plotted jets +rank0 [2026-11-15 05:51:41,395] INFO : Plotted jet ratio +rank0 [2026-11-15 05:51:41,405] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:51:42,854] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_46000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:51:43,309] INFO : Plotted number of elements +rank0 [2026-11-15 05:51:44,615] INFO : Plotted jets +rank0 [2026-11-15 05:51:45,610] INFO : Plotted jet ratio +rank0 [2026-11-15 05:53:42,324] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 05:53:42,325] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:55:39,877] INFO : Step 47000/500000 rank0 | Train Loss: 1.5613 | Train Loss Unweighted: 0.8124 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2241 | Regression_cos_phi: 0.0048 | Regression_energy: 0.2810 | Regression_eta: 0.0290 | Regression_pt: 0.2690 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0926 | Classification_binary: 0.2241 | Regression_cos_phi: 0.2821 | Regression_energy: 0.1682 | Regression_eta: 0.3773 | Regression_pt: 0.1609 | Regression_sin_phi: 0.2561 | LR: 1.27e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4507s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0564 | max_loss: 0.0029 | max_backward: 0.1175 | max_optimizer_step: 0.0835 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8724.1 | uss_MB: 7578.8 | child_rss_MB: 23698.7 | allocated_MB: 194.2 | reserved_MB: 19220.0 | inactive_split_MB: 219.8 +rank0 [2026-11-15 05:55:39,881] INFO : Running validation on rank0 +rank0 [2026-11-15 05:55:39,881] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:55:39,881] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:55:40,504] INFO : saved batch0_epoch47000.parquet +rank0 [2026-11-15 05:55:40,629] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:55:40,748] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:55:40,820] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:55:40,889] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:55:40,957] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:55:41,025] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:55:41,095] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:55:41,204] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:55:41,309] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:55:41,411] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:55:41,480] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:55:41,547] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:55:41,613] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:55:41,678] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:55:41,744] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:55:41,848] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:55:41,855] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:55:41,858] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:55:41,860] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:55:41,863] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:55:41,866] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:55:41,868] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:55:41,871] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:55:41,873] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:55:41,876] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:55:41,878] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:55:41,881] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:55:41,883] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:55:41,885] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:55:41,888] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:55:41,890] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:55:49,563] INFO : VALIDATION | Step=47000/500000 | Train Loss=1.5613 | Valid Loss=2.5871 | Train Loss Unweighted=0.8124 | Valid Loss Unweighted=0.8493 | Stale=8 | ETA=2090.4m +rank0 [2026-11-15 05:55:49,563] INFO : split_configs=['1'] +rank0 [2026-11-15 05:55:49,575] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 05:55:49,582] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 05:55:49,682] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:55:50,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 05:55:50,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 05:55:51,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 05:55:51,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 05:55:52,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 05:55:52,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 05:55:52,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 05:55:53,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 05:55:53,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 05:55:53,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 05:55:54,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 05:55:54,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 05:55:54,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 05:55:55,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 05:55:55,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 05:55:55,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 05:55:55,824] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 05:55:55,828] INFO : split_configs=['1'] +rank0 [2026-11-15 05:55:55,839] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 05:55:55,842] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 05:55:55,912] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:55:57,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:55:57,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:55:57,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:55:58,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:55:58,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:55:59,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:55:59,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:56:00,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:56:00,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:56:01,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:56:01,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:56:01,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:56:02,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:56:02,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:56:03,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:56:03,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:56:03,811] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 05:56:03,814] INFO : split_configs=['1'] +rank0 [2026-11-15 05:56:03,825] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 05:56:03,829] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:56:03,902] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:56:04,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 05:56:05,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 05:56:05,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 05:56:06,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 05:56:06,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 05:56:07,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 05:56:07,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 05:56:07,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 05:56:08,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 05:56:08,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 05:56:09,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 05:56:09,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 05:56:09,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 05:56:10,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 05:56:10,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 05:56:11,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 05:56:11,284] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 05:56:11,292] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf +rank0 [2026-11-15 05:56:12,451] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_qq_pf +rank0 [2026-11-15 05:56:12,876] INFO : Plotted number of elements +rank0 [2026-11-15 05:56:14,153] INFO : Plotted jets +rank0 [2026-11-15 05:56:15,667] INFO : Plotted jet ratio +rank0 [2026-11-15 05:56:15,678] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:56:17,311] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ttbar_pf +rank0 [2026-11-15 05:56:17,749] INFO : Plotted number of elements +rank0 [2026-11-15 05:56:19,083] INFO : Plotted jets +rank0 [2026-11-15 05:56:20,133] INFO : Plotted jet ratio +rank0 [2026-11-15 05:56:20,142] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:56:21,579] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_47000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 05:56:22,027] INFO : Plotted number of elements +rank0 [2026-11-15 05:56:23,342] INFO : Plotted jets +rank0 [2026-11-15 05:56:24,329] INFO : Plotted jet ratio +rank0 [2026-11-15 06:00:17,589] INFO : Step 48000/500000 rank0 | Train Loss: 1.8338 | Train Loss Unweighted: 0.8450 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2306 | Regression_cos_phi: 0.0069 | Regression_energy: 0.2920 | Regression_eta: 0.0203 | Regression_pt: 0.2872 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.2306 | Regression_cos_phi: 0.4067 | Regression_energy: 0.1748 | Regression_eta: 0.2639 | Regression_pt: 0.1718 | Regression_sin_phi: 0.4784 | LR: 1.31e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4919s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0025 | max_forward: 0.0683 | max_loss: 0.0033 | max_backward: 0.1495 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.8395 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8810.6 | uss_MB: 7665.3 | child_rss_MB: 23852.1 | allocated_MB: 196.4 | reserved_MB: 18934.0 | inactive_split_MB: 153.6 +rank0 [2026-11-15 06:00:17,591] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-48000.pth +rank0 [2026-11-15 06:00:17,659] INFO : Running validation on rank0 +rank0 [2026-11-15 06:00:17,659] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:00:17,659] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:00:18,281] INFO : saved batch0_epoch48000.parquet +rank0 [2026-11-15 06:00:18,403] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:00:18,517] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:00:18,588] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:00:18,655] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:00:18,723] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:00:18,792] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:00:18,860] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:00:18,973] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:00:19,082] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:00:19,189] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:00:19,258] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:00:19,325] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:00:19,390] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:00:19,456] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:00:19,524] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:00:19,625] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:00:19,632] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:00:19,635] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:00:19,637] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:00:19,640] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:00:19,643] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:00:19,645] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:00:19,647] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:00:19,650] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:00:19,652] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:00:19,655] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:00:19,657] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:00:19,660] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:00:19,662] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:00:19,665] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:00:19,667] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:00:27,462] INFO : VALIDATION | Step=48000/500000 | Train Loss=1.8338 | Valid Loss=2.5264 | Train Loss Unweighted=0.8450 | Valid Loss Unweighted=0.8470 | Stale=0 | ETA=2086.0m +rank0 [2026-11-15 06:00:27,463] INFO : split_configs=['1'] +rank0 [2026-11-15 06:00:27,475] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:00:27,482] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:00:27,583] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:00:28,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:00:28,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:00:29,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:00:29,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:00:29,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:00:30,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:00:30,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:00:30,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:00:31,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:00:31,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:00:31,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:00:32,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:00:32,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:00:32,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:00:33,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:00:33,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:00:33,724] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:00:33,727] INFO : split_configs=['1'] +rank0 [2026-11-15 06:00:33,737] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:00:33,741] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:00:33,811] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:00:34,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:00:35,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:00:35,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:00:36,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:00:36,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:00:37,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:00:37,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:00:38,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:00:38,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:00:38,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:00:39,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:00:39,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:00:40,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:00:40,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:00:41,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:00:41,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:00:41,671] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:00:41,674] INFO : split_configs=['1'] +rank0 [2026-11-15 06:00:41,721] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:00:41,725] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:00:41,791] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:00:42,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:00:43,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:00:43,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:00:44,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:00:44,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:00:44,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:00:45,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:00:45,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:00:46,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:00:46,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:00:46,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:00:47,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:00:47,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:00:48,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:00:48,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:00:49,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:00:49,138] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:00:49,147] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf +rank0 [2026-11-15 06:00:50,353] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_qq_pf +rank0 [2026-11-15 06:00:50,790] INFO : Plotted number of elements +rank0 [2026-11-15 06:00:52,076] INFO : Plotted jets +rank0 [2026-11-15 06:00:53,598] INFO : Plotted jet ratio +rank0 [2026-11-15 06:00:53,611] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:00:55,286] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:00:55,718] INFO : Plotted number of elements +rank0 [2026-11-15 06:00:57,066] INFO : Plotted jets +rank0 [2026-11-15 06:00:58,099] INFO : Plotted jet ratio +rank0 [2026-11-15 06:00:58,108] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:00:59,542] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_48000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:01:00,007] INFO : Plotted number of elements +rank0 [2026-11-15 06:01:01,319] INFO : Plotted jets +rank0 [2026-11-15 06:01:02,322] INFO : Plotted jet ratio +rank0 [2026-11-15 06:01:02,770] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 06:01:02,770] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:04:56,331] INFO : Step 49000/500000 rank0 | Train Loss: 2.5550 | Train Loss Unweighted: 0.8573 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2263 | Regression_cos_phi: 0.0082 | Regression_energy: 0.2902 | Regression_eta: 0.0781 | Regression_pt: 0.2476 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1039 | Classification_binary: 0.2263 | Regression_cos_phi: 0.4837 | Regression_energy: 0.1737 | Regression_eta: 1.0146 | Regression_pt: 0.1481 | Regression_sin_phi: 0.4048 | LR: 1.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3543s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0545 | max_loss: 0.0030 | max_backward: 0.1127 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.7458 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 8884.7 | uss_MB: 7739.4 | child_rss_MB: 23926.1 | allocated_MB: 193.4 | reserved_MB: 18790.0 | inactive_split_MB: 202.6 +rank0 [2026-11-15 06:04:56,333] INFO : Running validation on rank0 +rank0 [2026-11-15 06:04:56,333] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:04:56,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:04:56,969] INFO : saved batch0_epoch49000.parquet +rank0 [2026-11-15 06:04:57,092] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:04:57,206] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:04:57,275] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:04:57,342] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:04:57,408] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:04:57,476] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:04:57,544] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:04:57,656] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:04:57,764] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:04:57,870] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:04:57,939] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:04:58,005] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:04:58,072] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:04:58,140] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:04:58,207] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:04:58,309] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:04:58,317] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:04:58,319] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:04:58,322] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:04:58,324] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:04:58,327] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:04:58,330] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:04:58,333] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:04:58,336] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:04:58,339] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:04:58,341] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:04:58,345] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:04:58,347] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:04:58,349] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:04:58,352] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:04:58,354] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:05:06,066] INFO : VALIDATION | Step=49000/500000 | Train Loss=2.5550 | Valid Loss=2.6168 | Train Loss Unweighted=0.8573 | Valid Loss Unweighted=0.8523 | Stale=1 | ETA=2081.6m +rank0 [2026-11-15 06:05:06,067] INFO : split_configs=['1'] +rank0 [2026-11-15 06:05:06,076] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:05:06,084] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:05:06,184] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:05:07,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:05:07,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:05:07,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:05:08,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:05:08,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:05:08,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:05:09,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:05:09,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:05:09,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:05:10,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:05:10,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:05:10,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:05:11,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:05:11,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:05:11,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:05:12,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:05:12,294] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:05:12,296] INFO : split_configs=['1'] +rank0 [2026-11-15 06:05:12,308] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:05:12,312] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:05:12,383] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:05:13,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:05:13,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:05:14,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:05:14,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:05:15,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:05:15,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:05:16,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:05:16,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:05:17,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:05:17,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:05:17,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:05:18,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:05:18,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:05:19,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:05:19,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:05:20,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:05:20,203] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:05:20,206] INFO : split_configs=['1'] +rank0 [2026-11-15 06:05:20,217] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:05:20,220] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:05:20,294] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:05:21,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:05:21,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:05:22,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:05:22,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:05:23,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:05:23,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:05:23,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:05:24,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:05:24,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:05:25,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:05:25,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:05:25,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:05:26,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:05:26,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:05:27,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:05:27,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:05:27,634] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:05:27,641] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf +rank0 [2026-11-15 06:05:28,823] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_qq_pf +rank0 [2026-11-15 06:05:29,250] INFO : Plotted number of elements +rank0 [2026-11-15 06:05:30,525] INFO : Plotted jets +rank0 [2026-11-15 06:05:32,044] INFO : Plotted jet ratio +rank0 [2026-11-15 06:05:32,055] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:05:33,707] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:05:34,150] INFO : Plotted number of elements +rank0 [2026-11-15 06:05:35,482] INFO : Plotted jets +rank0 [2026-11-15 06:05:36,508] INFO : Plotted jet ratio +rank0 [2026-11-15 06:05:36,517] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:05:37,941] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_49000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:05:38,384] INFO : Plotted number of elements +rank0 [2026-11-15 06:05:39,688] INFO : Plotted jets +rank0 [2026-11-15 06:05:40,673] INFO : Plotted jet ratio +rank0 [2026-11-15 06:07:36,887] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 06:07:36,887] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:09:34,221] INFO : Step 50000/500000 rank0 | Train Loss: 2.4711 | Train Loss Unweighted: 0.9205 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2691 | Regression_cos_phi: 0.0094 | Regression_energy: 0.3114 | Regression_eta: 0.0361 | Regression_pt: 0.2824 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2691 | Regression_cos_phi: 0.5515 | Regression_energy: 0.1864 | Regression_eta: 0.4689 | Regression_pt: 0.1689 | Regression_sin_phi: 0.7447 | LR: 1.40e-04 | DataLoad Time: 0.0172s | Model Forward Time: 1.4133s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0024 | max_forward: 0.0540 | max_loss: 0.0031 | max_backward: 0.1127 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.8029 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 9111.1 | uss_MB: 7966.9 | child_rss_MB: 23837.2 | allocated_MB: 194.3 | reserved_MB: 17004.0 | inactive_split_MB: 265.7 +rank0 [2026-11-15 06:09:34,224] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-50000.pth +rank0 [2026-11-15 06:09:34,290] INFO : Running validation on rank0 +rank0 [2026-11-15 06:09:34,290] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:09:34,290] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:09:34,912] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-15 06:09:35,036] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:09:35,154] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:09:35,226] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:09:35,295] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:09:35,363] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:09:35,434] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:09:35,505] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:09:35,618] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:09:35,726] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:09:35,832] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:09:35,901] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:09:35,968] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:09:36,036] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:09:36,106] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:09:36,174] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:09:36,280] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:09:36,287] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:09:36,290] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:09:36,293] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:09:36,295] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:09:36,298] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:09:36,300] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:09:36,303] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:09:36,306] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:09:36,308] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:09:36,310] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:09:36,313] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:09:36,315] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:09:36,324] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:09:36,327] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:09:36,329] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:09:44,064] INFO : VALIDATION | Step=50000/500000 | Train Loss=2.4711 | Valid Loss=2.5399 | Train Loss Unweighted=0.9205 | Valid Loss Unweighted=0.8491 | Stale=2 | ETA=2077.2m +rank0 [2026-11-15 06:09:44,065] INFO : split_configs=['1'] +rank0 [2026-11-15 06:09:44,074] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:09:44,080] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:09:44,182] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:09:45,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:09:45,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:09:45,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:09:46,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:09:46,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:09:46,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:09:47,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:09:47,539] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:09:47,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:09:48,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:09:48,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:09:48,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:09:49,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:09:49,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:09:49,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:09:50,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:09:50,293] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:09:50,296] INFO : split_configs=['1'] +rank0 [2026-11-15 06:09:50,308] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:09:50,323] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:09:50,393] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:09:51,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:09:51,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:09:52,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:09:52,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:09:53,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:09:53,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:09:54,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:09:54,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:09:55,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:09:55,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:09:55,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:09:56,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:09:56,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:09:57,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:09:57,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:09:58,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:09:58,291] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:09:58,294] INFO : split_configs=['1'] +rank0 [2026-11-15 06:09:58,306] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:09:58,309] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:09:58,384] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:09:59,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:09:59,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:10:00,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:10:00,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:10:01,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:10:01,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:10:01,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:10:02,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:10:02,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:10:03,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:10:03,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:10:03,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:10:04,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:10:04,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:10:05,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:10:05,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:10:05,723] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:10:05,730] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf +rank0 [2026-11-15 06:10:06,901] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_qq_pf +rank0 [2026-11-15 06:10:07,324] INFO : Plotted number of elements +rank0 [2026-11-15 06:10:08,600] INFO : Plotted jets +rank0 [2026-11-15 06:10:10,115] INFO : Plotted jet ratio +rank0 [2026-11-15 06:10:10,126] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:10:11,751] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:10:12,182] INFO : Plotted number of elements +rank0 [2026-11-15 06:10:13,525] INFO : Plotted jets +rank0 [2026-11-15 06:10:14,542] INFO : Plotted jet ratio +rank0 [2026-11-15 06:10:14,551] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:10:16,010] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_50000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:10:16,469] INFO : Plotted number of elements +rank0 [2026-11-15 06:10:17,788] INFO : Plotted jets +rank0 [2026-11-15 06:10:18,770] INFO : Plotted jet ratio +rank0 [2026-11-15 06:14:15,108] INFO : Step 51000/500000 rank0 | Train Loss: 2.0990 | Train Loss Unweighted: 0.8173 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2388 | Regression_cos_phi: 0.0056 | Regression_energy: 0.2687 | Regression_eta: 0.0305 | Regression_pt: 0.2619 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0975 | Classification_binary: 0.2388 | Regression_cos_phi: 0.3279 | Regression_energy: 0.1608 | Regression_eta: 0.3956 | Regression_pt: 0.1566 | Regression_sin_phi: 0.7216 | LR: 1.44e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4773s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0022 | max_forward: 0.0509 | max_loss: 0.0027 | max_backward: 0.1086 | max_optimizer_step: 0.0875 | max_tensorboard_logging: 0.8651 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9187.4 | uss_MB: 8043.1 | child_rss_MB: 23808.6 | allocated_MB: 193.8 | reserved_MB: 21746.0 | inactive_split_MB: 168.2 +rank0 [2026-11-15 06:14:15,158] INFO : Running validation on rank0 +rank0 [2026-11-15 06:14:15,158] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:14:15,158] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:14:15,825] INFO : saved batch0_epoch51000.parquet +rank0 [2026-11-15 06:14:15,947] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:14:16,086] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:14:16,157] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:14:16,224] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:14:16,290] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:14:16,357] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:14:16,423] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:14:16,533] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:14:16,638] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:14:16,740] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:14:16,808] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:14:16,875] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:14:16,939] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:14:17,005] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:14:17,075] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:14:17,177] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:14:17,197] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:14:17,210] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:14:17,213] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:14:17,277] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:14:17,286] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:14:17,288] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:14:17,291] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:14:17,306] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:14:17,308] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:14:17,310] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:14:17,313] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:14:17,315] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:14:17,318] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:14:17,321] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:14:17,323] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:14:25,056] INFO : VALIDATION | Step=51000/500000 | Train Loss=2.0990 | Valid Loss=2.6327 | Train Loss Unweighted=0.8173 | Valid Loss Unweighted=0.8565 | Stale=3 | ETA=2073.1m +rank0 [2026-11-15 06:14:25,056] INFO : split_configs=['1'] +rank0 [2026-11-15 06:14:25,068] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:14:25,089] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:14:25,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:14:26,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:14:26,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:14:26,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:14:27,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:14:27,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:14:27,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:14:28,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:14:28,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:14:28,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:14:29,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:14:29,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:14:29,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:14:30,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:14:30,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:14:30,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:14:31,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:14:31,292] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:14:31,295] INFO : split_configs=['1'] +rank0 [2026-11-15 06:14:31,306] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:14:31,309] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:14:31,375] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:14:32,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:14:32,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:14:33,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:14:33,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:14:34,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:14:34,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:14:35,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:14:35,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:14:36,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:14:36,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:14:36,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:14:37,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:14:37,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:14:38,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:14:38,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:14:39,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:14:39,283] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:14:39,287] INFO : split_configs=['1'] +rank0 [2026-11-15 06:14:39,298] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:14:39,301] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:14:39,376] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:14:40,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:14:40,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:14:41,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:14:41,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:14:42,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:14:42,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:14:42,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:14:43,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:14:43,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:14:44,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:14:44,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:14:45,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:14:45,474] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:14:45,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:14:46,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:14:46,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:14:46,803] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:14:46,811] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf +rank0 [2026-11-15 06:14:48,006] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_qq_pf +rank0 [2026-11-15 06:14:48,439] INFO : Plotted number of elements +rank0 [2026-11-15 06:14:49,731] INFO : Plotted jets +rank0 [2026-11-15 06:14:51,247] INFO : Plotted jet ratio +rank0 [2026-11-15 06:14:51,257] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:14:52,903] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:14:53,336] INFO : Plotted number of elements +rank0 [2026-11-15 06:14:54,678] INFO : Plotted jets +rank0 [2026-11-15 06:14:55,704] INFO : Plotted jet ratio +rank0 [2026-11-15 06:14:55,715] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:14:57,219] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_51000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:14:57,681] INFO : Plotted number of elements +rank0 [2026-11-15 06:14:58,993] INFO : Plotted jets +rank0 [2026-11-15 06:14:59,979] INFO : Plotted jet ratio +rank0 [2026-11-15 06:15:00,431] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 06:15:00,431] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:18:53,917] INFO : Step 52000/500000 rank0 | Train Loss: 2.1793 | Train Loss Unweighted: 0.7986 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2479 | Regression_cos_phi: 0.0093 | Regression_energy: 0.2545 | Regression_eta: 0.0338 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.2479 | Regression_cos_phi: 0.5447 | Regression_energy: 0.1523 | Regression_eta: 0.4395 | Regression_pt: 0.1458 | Regression_sin_phi: 0.5615 | LR: 1.49e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4872s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0608 | max_loss: 0.0031 | max_backward: 0.1280 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.8486 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9116.0 | uss_MB: 7971.8 | child_rss_MB: 23839.6 | allocated_MB: 195.9 | reserved_MB: 20338.0 | inactive_split_MB: 178.1 +rank0 [2026-11-15 06:18:53,919] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-52000.pth +rank0 [2026-11-15 06:18:53,984] INFO : Running validation on rank0 +rank0 [2026-11-15 06:18:53,986] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:18:53,986] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:18:54,613] INFO : saved batch0_epoch52000.parquet +rank0 [2026-11-15 06:18:54,734] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:18:54,847] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:18:54,917] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:18:54,986] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:18:55,054] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:18:55,125] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:18:55,194] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:18:55,305] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:18:55,411] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:18:55,514] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:18:55,581] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:18:55,647] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:18:55,712] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:18:55,778] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:18:55,844] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:18:55,944] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:18:55,952] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:18:55,954] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:18:55,957] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:18:55,959] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:18:55,962] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:18:55,964] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:18:55,967] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:18:55,969] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:18:55,972] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:18:55,974] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:18:55,977] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:18:55,979] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:18:55,981] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:18:55,984] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:18:55,986] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:19:03,688] INFO : VALIDATION | Step=52000/500000 | Train Loss=2.1793 | Valid Loss=2.5885 | Train Loss Unweighted=0.7986 | Valid Loss Unweighted=0.8498 | Stale=4 | ETA=2068.7m +rank0 [2026-11-15 06:19:03,689] INFO : split_configs=['1'] +rank0 [2026-11-15 06:19:03,700] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:19:03,707] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:19:03,831] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:19:04,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:19:05,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:19:05,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:19:05,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:19:06,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:19:06,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:19:06,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:19:07,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:19:07,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:19:07,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:19:08,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:19:08,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:19:08,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:19:09,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:19:09,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:19:10,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:19:10,113] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:19:10,117] INFO : split_configs=['1'] +rank0 [2026-11-15 06:19:10,128] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:19:10,131] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:19:10,208] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:19:11,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:19:11,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:19:12,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:19:12,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:19:13,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:19:13,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:19:13,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:19:14,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:19:14,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:19:15,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:19:15,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:19:16,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:19:16,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:19:17,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:19:17,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:19:18,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:19:18,121] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:19:18,124] INFO : split_configs=['1'] +rank0 [2026-11-15 06:19:18,137] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:19:18,140] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:19:18,214] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:19:19,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:19:19,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:19:20,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:19:20,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:19:20,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:19:21,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:19:21,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:19:22,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:19:22,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:19:23,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:19:23,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:19:23,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:19:24,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:19:24,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:19:25,099] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:19:25,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:19:25,642] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:19:25,649] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf +rank0 [2026-11-15 06:19:26,821] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_qq_pf +rank0 [2026-11-15 06:19:27,249] INFO : Plotted number of elements +rank0 [2026-11-15 06:19:28,540] INFO : Plotted jets +rank0 [2026-11-15 06:19:30,099] INFO : Plotted jet ratio +rank0 [2026-11-15 06:19:30,111] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:19:31,717] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:19:32,160] INFO : Plotted number of elements +rank0 [2026-11-15 06:19:33,507] INFO : Plotted jets +rank0 [2026-11-15 06:19:34,540] INFO : Plotted jet ratio +rank0 [2026-11-15 06:19:34,550] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:19:36,022] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_52000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:19:36,489] INFO : Plotted number of elements +rank0 [2026-11-15 06:19:37,806] INFO : Plotted jets +rank0 [2026-11-15 06:19:38,805] INFO : Plotted jet ratio +rank0 [2026-11-15 06:21:35,268] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 06:21:35,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:23:32,724] INFO : Step 53000/500000 rank0 | Train Loss: 1.6658 | Train Loss Unweighted: 0.7735 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2311 | Regression_cos_phi: 0.0072 | Regression_energy: 0.2533 | Regression_eta: 0.0253 | Regression_pt: 0.2518 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0999 | Classification_binary: 0.2311 | Regression_cos_phi: 0.4240 | Regression_energy: 0.1516 | Regression_eta: 0.3286 | Regression_pt: 0.1506 | Regression_sin_phi: 0.2801 | LR: 1.53e-04 | DataLoad Time: 0.0144s | Model Forward Time: 1.4999s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0632 | max_loss: 0.0029 | max_backward: 0.1300 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9177.3 | uss_MB: 8033.0 | child_rss_MB: 23700.9 | allocated_MB: 197.1 | reserved_MB: 19812.0 | inactive_split_MB: 230.9 +rank0 [2026-11-15 06:23:32,729] INFO : Running validation on rank0 +rank0 [2026-11-15 06:23:32,730] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:23:32,730] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:23:33,361] INFO : saved batch0_epoch53000.parquet +rank0 [2026-11-15 06:23:33,483] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:23:33,595] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:23:33,665] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:23:33,734] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:23:33,801] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:23:33,870] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:23:33,938] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:23:34,051] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:23:34,157] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:23:34,260] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:23:34,327] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:23:34,393] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:23:34,458] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:23:34,523] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:23:34,588] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:23:34,690] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:23:34,697] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:23:34,700] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:23:34,703] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:23:34,705] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:23:34,708] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:23:34,710] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:23:34,713] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:23:34,715] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:23:34,718] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:23:34,720] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:23:34,723] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:23:34,725] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:23:34,728] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:23:34,730] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:23:34,733] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:23:42,400] INFO : VALIDATION | Step=53000/500000 | Train Loss=1.6658 | Valid Loss=2.6057 | Train Loss Unweighted=0.7735 | Valid Loss Unweighted=0.8462 | Stale=5 | ETA=2064.4m +rank0 [2026-11-15 06:23:42,401] INFO : split_configs=['1'] +rank0 [2026-11-15 06:23:42,413] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:23:42,420] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:23:42,547] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:23:43,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:23:43,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:23:44,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:23:44,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:23:44,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:23:45,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:23:45,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:23:45,890] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:23:46,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:23:46,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:23:46,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:23:47,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:23:47,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:23:47,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:23:48,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:23:48,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:23:48,663] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:23:48,665] INFO : split_configs=['1'] +rank0 [2026-11-15 06:23:48,676] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:23:48,679] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:23:48,751] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:23:49,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:23:50,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:23:50,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:23:51,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:23:51,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:23:52,107] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:23:52,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:23:53,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:23:53,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:23:53,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:23:54,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:23:54,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:23:55,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:23:55,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:23:56,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:23:56,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:23:56,714] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:23:56,717] INFO : split_configs=['1'] +rank0 [2026-11-15 06:23:56,728] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:23:56,731] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:23:56,807] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:23:57,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:23:58,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:23:58,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:23:59,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:23:59,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:23:59,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:24:00,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:24:00,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:24:01,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:24:01,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:24:01,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:24:02,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:24:02,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:24:03,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:24:03,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:24:04,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:24:04,172] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:24:04,179] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf +rank0 [2026-11-15 06:24:05,363] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_qq_pf +rank0 [2026-11-15 06:24:05,798] INFO : Plotted number of elements +rank0 [2026-11-15 06:24:07,073] INFO : Plotted jets +rank0 [2026-11-15 06:24:08,612] INFO : Plotted jet ratio +rank0 [2026-11-15 06:24:08,624] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:24:10,216] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:24:10,678] INFO : Plotted number of elements +rank0 [2026-11-15 06:24:12,047] INFO : Plotted jets +rank0 [2026-11-15 06:24:13,101] INFO : Plotted jet ratio +rank0 [2026-11-15 06:24:13,110] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:24:14,551] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_53000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:24:14,986] INFO : Plotted number of elements +rank0 [2026-11-15 06:24:16,300] INFO : Plotted jets +rank0 [2026-11-15 06:24:17,280] INFO : Plotted jet ratio +rank0 [2026-11-15 06:28:10,909] INFO : Step 54000/500000 rank0 | Train Loss: 1.9095 | Train Loss Unweighted: 0.7506 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2291 | Regression_cos_phi: 0.0071 | Regression_energy: 0.2409 | Regression_eta: 0.0155 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.2291 | Regression_cos_phi: 0.4191 | Regression_energy: 0.1442 | Regression_eta: 0.2019 | Regression_pt: 0.1476 | Regression_sin_phi: 0.6861 | LR: 1.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5580s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0722 | max_loss: 0.0028 | max_backward: 0.1554 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9339.3 | uss_MB: 8195.1 | child_rss_MB: 23782.2 | allocated_MB: 199.3 | reserved_MB: 19972.0 | inactive_split_MB: 314.7 +rank0 [2026-11-15 06:28:10,913] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-54000.pth +rank0 [2026-11-15 06:28:10,984] INFO : Running validation on rank0 +rank0 [2026-11-15 06:28:10,984] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:28:10,984] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:28:11,614] INFO : saved batch0_epoch54000.parquet +rank0 [2026-11-15 06:28:11,741] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:28:11,857] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:28:11,929] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:28:11,997] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:28:12,068] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:28:12,139] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:28:12,208] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:28:12,319] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:28:12,423] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:28:12,527] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:28:12,596] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:28:12,664] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:28:12,729] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:28:12,796] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:28:12,862] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:28:12,965] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:28:12,972] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:28:12,975] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:28:12,978] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:28:12,980] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:28:12,983] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:28:12,985] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:28:12,988] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:28:12,990] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:28:12,993] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:28:12,995] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:28:12,998] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:28:13,000] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:28:13,002] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:28:13,005] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:28:13,007] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:28:20,681] INFO : VALIDATION | Step=54000/500000 | Train Loss=1.9095 | Valid Loss=2.5306 | Train Loss Unweighted=0.7506 | Valid Loss Unweighted=0.8396 | Stale=6 | ETA=2059.9m +rank0 [2026-11-15 06:28:20,682] INFO : split_configs=['1'] +rank0 [2026-11-15 06:28:20,693] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:28:20,701] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:28:20,793] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:28:21,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:28:22,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:28:22,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:28:22,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:28:23,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:28:23,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:28:23,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:28:24,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:28:24,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:28:24,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:28:25,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:28:25,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:28:25,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:28:26,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:28:26,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:28:26,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:28:26,906] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:28:26,911] INFO : split_configs=['1'] +rank0 [2026-11-15 06:28:26,922] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:28:26,925] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:28:26,995] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:28:28,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:28:28,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:28:29,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:28:29,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:28:29,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:28:30,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:28:30,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:28:31,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:28:31,763] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:28:32,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:28:32,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:28:33,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:28:33,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:28:34,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:28:34,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:28:34,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:28:34,983] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:28:34,986] INFO : split_configs=['1'] +rank0 [2026-11-15 06:28:34,997] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:28:35,001] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:28:35,073] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:28:36,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:28:36,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:28:37,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:28:37,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:28:37,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:28:38,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:28:38,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:28:39,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:28:39,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:28:39,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:28:40,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:28:40,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:28:41,171] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:28:41,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:28:41,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:28:42,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:28:42,514] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:28:42,522] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf +rank0 [2026-11-15 06:28:44,197] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_qq_pf +rank0 [2026-11-15 06:28:44,634] INFO : Plotted number of elements +rank0 [2026-11-15 06:28:45,931] INFO : Plotted jets +rank0 [2026-11-15 06:28:47,489] INFO : Plotted jet ratio +rank0 [2026-11-15 06:28:47,505] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:28:49,201] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:28:49,635] INFO : Plotted number of elements +rank0 [2026-11-15 06:28:50,990] INFO : Plotted jets +rank0 [2026-11-15 06:28:52,022] INFO : Plotted jet ratio +rank0 [2026-11-15 06:28:52,032] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:28:53,515] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_54000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:28:53,961] INFO : Plotted number of elements +rank0 [2026-11-15 06:28:55,289] INFO : Plotted jets +rank0 [2026-11-15 06:28:56,278] INFO : Plotted jet ratio +rank0 [2026-11-15 06:28:56,746] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 06:28:56,746] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:32:50,384] INFO : Step 55000/500000 rank0 | Train Loss: 1.8708 | Train Loss Unweighted: 0.8813 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2422 | Regression_cos_phi: 0.0060 | Regression_energy: 0.3065 | Regression_eta: 0.0306 | Regression_pt: 0.2888 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0905 | Classification_binary: 0.2422 | Regression_cos_phi: 0.3545 | Regression_energy: 0.1834 | Regression_eta: 0.3980 | Regression_pt: 0.1727 | Regression_sin_phi: 0.4294 | LR: 1.62e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5283s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0582 | max_loss: 0.0032 | max_backward: 0.1229 | max_optimizer_step: 0.0893 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9460.4 | uss_MB: 8316.2 | child_rss_MB: 23820.0 | allocated_MB: 197.8 | reserved_MB: 18872.0 | inactive_split_MB: 202.2 +rank0 [2026-11-15 06:32:50,388] INFO : Running validation on rank0 +rank0 [2026-11-15 06:32:50,388] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:32:50,388] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:32:51,013] INFO : saved batch0_epoch55000.parquet +rank0 [2026-11-15 06:32:51,139] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:32:51,253] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:32:51,325] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:32:51,394] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:32:51,463] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:32:51,534] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:32:51,604] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:32:51,716] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:32:51,824] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:32:51,928] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:32:51,996] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:32:52,065] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:32:52,132] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:32:52,198] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:32:52,266] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:32:52,368] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:32:52,375] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:32:52,378] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:32:52,380] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:32:52,383] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:32:52,386] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:32:52,388] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:32:52,390] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:32:52,393] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:32:52,395] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:32:52,398] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:32:52,401] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:32:52,403] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:32:52,405] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:32:52,409] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:32:52,411] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:33:00,132] INFO : VALIDATION | Step=55000/500000 | Train Loss=1.8708 | Valid Loss=2.5842 | Train Loss Unweighted=0.8813 | Valid Loss Unweighted=0.8403 | Stale=7 | ETA=2055.6m +rank0 [2026-11-15 06:33:00,133] INFO : split_configs=['1'] +rank0 [2026-11-15 06:33:00,145] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:33:00,163] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:33:00,272] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:33:01,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:33:01,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:33:01,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:33:02,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:33:02,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:33:02,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:33:03,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:33:03,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:33:03,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:33:04,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:33:04,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:33:05,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:33:05,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:33:05,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:33:05,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:33:06,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:33:06,403] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-15 06:33:06,406] INFO : split_configs=['1'] +rank0 [2026-11-15 06:33:06,417] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:33:06,422] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:33:06,498] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:33:07,656] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:33:08,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:33:08,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:33:08,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:33:09,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:33:09,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:33:10,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:33:10,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:33:11,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:33:11,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:33:12,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:33:12,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:33:12,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:33:13,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:33:13,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:33:14,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:33:14,369] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:33:14,373] INFO : split_configs=['1'] +rank0 [2026-11-15 06:33:14,384] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:33:14,388] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:33:14,465] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:33:15,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:33:16,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:33:16,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:33:16,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:33:17,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:33:17,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:33:18,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:33:18,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:33:18,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:33:19,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:33:19,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:33:20,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:33:20,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:33:20,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:33:21,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:33:21,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:33:21,900] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:33:21,906] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf +rank0 [2026-11-15 06:33:23,066] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_qq_pf +rank0 [2026-11-15 06:33:23,498] INFO : Plotted number of elements +rank0 [2026-11-15 06:33:24,789] INFO : Plotted jets +rank0 [2026-11-15 06:33:26,356] INFO : Plotted jet ratio +rank0 [2026-11-15 06:33:26,368] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:33:28,000] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:33:28,444] INFO : Plotted number of elements +rank0 [2026-11-15 06:33:29,798] INFO : Plotted jets +rank0 [2026-11-15 06:33:30,847] INFO : Plotted jet ratio +rank0 [2026-11-15 06:33:30,856] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:33:32,311] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_55000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:33:32,770] INFO : Plotted number of elements +rank0 [2026-11-15 06:33:34,094] INFO : Plotted jets +rank0 [2026-11-15 06:33:35,101] INFO : Plotted jet ratio +rank0 [2026-11-15 06:35:31,897] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 06:35:31,898] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:37:29,890] INFO : Step 56000/500000 rank0 | Train Loss: 2.1666 | Train Loss Unweighted: 0.9249 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2473 | Regression_cos_phi: 0.0061 | Regression_energy: 0.3136 | Regression_eta: 0.0291 | Regression_pt: 0.3170 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2473 | Regression_cos_phi: 0.3573 | Regression_energy: 0.1877 | Regression_eta: 0.3782 | Regression_pt: 0.1896 | Regression_sin_phi: 0.7318 | LR: 1.67e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5137s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0026 | max_forward: 0.0598 | max_loss: 0.0029 | max_backward: 0.1263 | max_optimizer_step: 0.0878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9460.5 | uss_MB: 8316.2 | child_rss_MB: 23703.6 | allocated_MB: 197.0 | reserved_MB: 22324.0 | inactive_split_MB: 209.0 +rank0 [2026-11-15 06:37:29,892] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-56000.pth +rank0 [2026-11-15 06:37:29,960] INFO : Running validation on rank0 +rank0 [2026-11-15 06:37:29,960] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:37:29,960] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:37:30,583] INFO : saved batch0_epoch56000.parquet +rank0 [2026-11-15 06:37:30,706] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:37:30,823] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:37:30,896] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:37:30,965] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:37:31,033] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:37:31,102] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:37:31,170] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:37:31,281] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:37:31,385] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:37:31,488] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:37:31,556] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:37:31,623] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:37:31,689] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:37:31,755] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:37:31,822] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:37:31,925] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:37:31,932] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:37:31,935] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:37:31,938] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:37:31,941] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:37:31,944] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:37:31,946] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:37:31,949] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:37:31,951] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:37:31,954] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:37:31,982] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:37:31,984] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:37:31,986] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:37:31,987] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:37:31,988] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:37:31,989] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:37:39,735] INFO : VALIDATION | Step=56000/500000 | Train Loss=2.1666 | Valid Loss=2.5654 | Train Loss Unweighted=0.9249 | Valid Loss Unweighted=0.8452 | Stale=8 | ETA=2051.3m +rank0 [2026-11-15 06:37:39,736] INFO : split_configs=['1'] +rank0 [2026-11-15 06:37:39,748] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:37:39,759] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:37:39,869] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:37:40,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:37:41,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:37:41,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:37:41,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:37:42,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:37:42,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:37:42,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:37:43,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:37:43,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:37:43,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:37:44,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:37:44,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:37:44,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:37:45,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:37:45,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:37:45,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:37:46,083] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:37:46,086] INFO : split_configs=['1'] +rank0 [2026-11-15 06:37:46,097] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:37:46,100] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:37:46,176] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:37:47,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:37:47,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:37:48,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:37:48,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:37:49,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:37:49,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:37:49,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:37:50,429] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:37:50,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:37:51,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:37:51,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:37:52,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:37:52,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:37:53,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:37:53,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:37:54,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:37:54,175] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:37:54,179] INFO : split_configs=['1'] +rank0 [2026-11-15 06:37:54,191] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:37:54,194] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:37:54,271] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:37:55,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:37:55,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:37:56,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:37:56,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:37:57,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:37:57,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:37:57,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:37:58,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:37:58,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:37:59,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:37:59,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:37:59,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:38:00,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:38:00,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:38:01,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:38:01,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:38:01,648] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:38:01,657] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf +rank0 [2026-11-15 06:38:02,814] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_qq_pf +rank0 [2026-11-15 06:38:03,237] INFO : Plotted number of elements +rank0 [2026-11-15 06:38:04,532] INFO : Plotted jets +rank0 [2026-11-15 06:38:06,108] INFO : Plotted jet ratio +rank0 [2026-11-15 06:38:06,119] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:38:07,714] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:38:08,154] INFO : Plotted number of elements +rank0 [2026-11-15 06:38:09,509] INFO : Plotted jets +rank0 [2026-11-15 06:38:10,536] INFO : Plotted jet ratio +rank0 [2026-11-15 06:38:10,547] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:38:11,981] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_56000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:38:12,429] INFO : Plotted number of elements +rank0 [2026-11-15 06:38:13,754] INFO : Plotted jets +rank0 [2026-11-15 06:38:14,743] INFO : Plotted jet ratio +rank0 [2026-11-15 06:42:08,250] INFO : Step 57000/500000 rank0 | Train Loss: 1.5645 | Train Loss Unweighted: 0.7050 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2324 | Regression_cos_phi: 0.0084 | Regression_energy: 0.2249 | Regression_eta: 0.0161 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0948 | Classification_binary: 0.2324 | Regression_cos_phi: 0.4929 | Regression_energy: 0.1346 | Regression_eta: 0.2088 | Regression_pt: 0.1307 | Regression_sin_phi: 0.2702 | LR: 1.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7355s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0646 | max_loss: 0.0027 | max_backward: 0.1373 | max_optimizer_step: 0.0886 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9650.0 | uss_MB: 8505.8 | child_rss_MB: 23704.7 | allocated_MB: 200.7 | reserved_MB: 21426.0 | inactive_split_MB: 187.3 +rank0 [2026-11-15 06:42:08,253] INFO : Running validation on rank0 +rank0 [2026-11-15 06:42:08,254] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:42:08,254] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:42:08,889] INFO : saved batch0_epoch57000.parquet +rank0 [2026-11-15 06:42:09,015] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:42:09,132] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:42:09,204] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:42:09,272] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:42:09,338] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:42:09,405] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:42:09,471] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:42:09,583] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:42:09,691] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:42:09,797] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:42:09,868] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:42:09,937] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:42:10,006] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:42:10,077] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:42:10,146] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:42:10,253] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:42:10,260] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:42:10,263] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:42:10,265] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:42:10,268] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:42:10,271] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:42:10,273] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:42:10,275] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:42:10,278] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:42:10,280] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:42:10,283] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:42:10,285] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:42:10,288] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:42:10,290] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:42:10,293] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:42:10,295] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:42:17,968] INFO : VALIDATION | Step=57000/500000 | Train Loss=1.5645 | Valid Loss=2.5628 | Train Loss Unweighted=0.7050 | Valid Loss Unweighted=0.8422 | Stale=9 | ETA=2046.8m +rank0 [2026-11-15 06:42:17,969] INFO : split_configs=['1'] +rank0 [2026-11-15 06:42:17,979] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:42:17,987] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:42:18,087] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:42:19,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:42:19,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:42:19,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:42:20,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:42:20,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:42:20,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:42:21,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:42:21,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:42:21,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:42:22,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:42:22,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:42:22,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:42:23,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:42:23,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:42:23,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:42:24,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:42:24,176] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:42:24,181] INFO : split_configs=['1'] +rank0 [2026-11-15 06:42:24,192] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:42:24,195] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:42:24,265] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:42:25,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:42:25,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:42:26,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:42:26,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:42:27,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:42:27,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:42:28,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:42:28,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:42:28,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:42:29,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:42:29,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:42:30,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:42:30,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:42:31,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:42:31,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:42:32,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:42:32,164] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:42:32,167] INFO : split_configs=['1'] +rank0 [2026-11-15 06:42:32,179] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:42:32,182] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:42:32,254] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:42:33,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:42:33,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:42:34,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:42:34,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:42:35,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:42:35,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:42:35,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:42:36,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:42:36,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:42:37,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:42:37,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:42:37,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:42:38,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:42:38,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:42:39,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:42:39,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:42:39,642] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:42:39,649] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf +rank0 [2026-11-15 06:42:40,854] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_qq_pf +rank0 [2026-11-15 06:42:41,289] INFO : Plotted number of elements +rank0 [2026-11-15 06:42:42,594] INFO : Plotted jets +rank0 [2026-11-15 06:42:44,146] INFO : Plotted jet ratio +rank0 [2026-11-15 06:42:44,157] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:42:45,843] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:42:46,281] INFO : Plotted number of elements +rank0 [2026-11-15 06:42:47,643] INFO : Plotted jets +rank0 [2026-11-15 06:42:48,676] INFO : Plotted jet ratio +rank0 [2026-11-15 06:42:48,685] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:42:50,113] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_57000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:42:50,580] INFO : Plotted number of elements +rank0 [2026-11-15 06:42:51,902] INFO : Plotted jets +rank0 [2026-11-15 06:42:52,892] INFO : Plotted jet ratio +rank0 [2026-11-15 06:42:53,364] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 06:42:53,364] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:46:47,590] INFO : Step 58000/500000 rank0 | Train Loss: 1.8553 | Train Loss Unweighted: 0.7971 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2124 | Regression_cos_phi: 0.0073 | Regression_energy: 0.2679 | Regression_eta: 0.0231 | Regression_pt: 0.2781 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0817 | Classification_binary: 0.2124 | Regression_cos_phi: 0.4296 | Regression_energy: 0.1603 | Regression_eta: 0.2995 | Regression_pt: 0.1663 | Regression_sin_phi: 0.5055 | LR: 1.76e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5340s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0633 | max_loss: 0.0030 | max_backward: 0.1339 | max_optimizer_step: 0.0897 | max_tensorboard_logging: 0.8589 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9597.2 | uss_MB: 8453.0 | child_rss_MB: 23815.2 | allocated_MB: 198.0 | reserved_MB: 19232.0 | inactive_split_MB: 648.0 +rank0 [2026-11-15 06:46:47,592] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-58000.pth +rank0 [2026-11-15 06:46:47,659] INFO : Running validation on rank0 +rank0 [2026-11-15 06:46:47,659] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:46:47,659] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:46:48,282] INFO : saved batch0_epoch58000.parquet +rank0 [2026-11-15 06:46:48,404] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:46:48,522] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:46:48,594] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:46:48,662] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:46:48,729] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:46:48,796] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:46:48,864] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:46:48,971] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:46:49,080] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:46:49,183] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:46:49,251] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:46:49,318] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:46:49,384] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:46:49,451] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:46:49,518] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:46:49,621] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:46:49,666] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:46:49,670] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:46:49,672] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:46:49,675] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:46:49,677] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:46:49,680] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:46:49,682] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:46:49,685] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:46:49,687] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:46:49,689] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:46:49,692] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:46:49,694] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:46:49,696] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:46:49,699] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:46:49,701] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:46:57,418] INFO : VALIDATION | Step=58000/500000 | Train Loss=1.8553 | Valid Loss=2.6198 | Train Loss Unweighted=0.7971 | Valid Loss Unweighted=0.8577 | Stale=10 | ETA=2042.5m +rank0 [2026-11-15 06:46:57,418] INFO : split_configs=['1'] +rank0 [2026-11-15 06:46:57,429] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:46:57,437] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:46:57,543] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:46:58,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:46:58,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:46:59,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:46:59,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:46:59,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:47:00,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:47:00,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:47:00,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:47:01,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:47:01,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:47:01,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:47:02,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:47:02,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:47:02,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:47:03,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:47:03,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:47:03,757] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:47:03,760] INFO : split_configs=['1'] +rank0 [2026-11-15 06:47:03,771] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:47:03,774] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:47:03,852] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:47:05,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:47:05,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:47:05,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:47:06,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:47:06,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:47:07,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:47:07,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:47:08,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:47:08,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:47:09,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:47:09,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:47:09,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:47:10,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:47:10,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:47:11,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:47:11,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:47:11,821] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:47:11,825] INFO : split_configs=['1'] +rank0 [2026-11-15 06:47:11,836] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:47:11,840] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:47:11,918] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:47:13,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:47:13,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:47:13,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:47:14,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:47:14,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:47:15,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:47:15,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:47:15,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:47:16,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:47:16,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:47:17,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:47:17,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:47:17,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:47:18,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:47:18,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:47:19,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:47:19,275] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:47:19,282] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf +rank0 [2026-11-15 06:47:20,497] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_qq_pf +rank0 [2026-11-15 06:47:20,918] INFO : Plotted number of elements +rank0 [2026-11-15 06:47:22,205] INFO : Plotted jets +rank0 [2026-11-15 06:47:23,751] INFO : Plotted jet ratio +rank0 [2026-11-15 06:47:23,763] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:47:25,429] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:47:25,857] INFO : Plotted number of elements +rank0 [2026-11-15 06:47:27,207] INFO : Plotted jets +rank0 [2026-11-15 06:47:28,230] INFO : Plotted jet ratio +rank0 [2026-11-15 06:47:28,239] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:47:29,700] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_58000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:47:30,156] INFO : Plotted number of elements +rank0 [2026-11-15 06:47:31,471] INFO : Plotted jets +rank0 [2026-11-15 06:47:32,465] INFO : Plotted jet ratio +rank0 [2026-11-15 06:49:28,624] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 06:49:28,624] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 06:51:25,612] INFO : Step 59000/500000 rank0 | Train Loss: 1.6872 | Train Loss Unweighted: 0.7391 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2432 | Regression_cos_phi: 0.0062 | Regression_energy: 0.2427 | Regression_eta: 0.0205 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.2432 | Regression_cos_phi: 0.3649 | Regression_energy: 0.1452 | Regression_eta: 0.2664 | Regression_pt: 0.1309 | Regression_sin_phi: 0.4646 | LR: 1.81e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5419s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0643 | max_loss: 0.0029 | max_backward: 0.1362 | max_optimizer_step: 0.0886 | max_tensorboard_logging: 0.8727 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9525.7 | uss_MB: 8381.4 | child_rss_MB: 23912.1 | allocated_MB: 198.5 | reserved_MB: 19170.0 | inactive_split_MB: 399.5 +rank0 [2026-11-15 06:51:25,616] INFO : Running validation on rank0 +rank0 [2026-11-15 06:51:25,616] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:51:25,616] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:51:26,239] INFO : saved batch0_epoch59000.parquet +rank0 [2026-11-15 06:51:26,359] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:51:26,474] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:51:26,545] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:51:26,613] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:51:26,681] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:51:26,753] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:51:26,823] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:51:26,935] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:51:27,040] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:51:27,145] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:51:27,213] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:51:27,280] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:51:27,345] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:51:27,412] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:51:27,478] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:51:27,580] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:51:27,587] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:51:27,590] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:51:27,593] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:51:27,595] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:51:27,598] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:51:27,600] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:51:27,603] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:51:27,605] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:51:27,608] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:51:27,610] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:51:27,613] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:51:27,615] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:51:27,617] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:51:27,620] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:51:27,622] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:51:35,393] INFO : VALIDATION | Step=59000/500000 | Train Loss=1.6872 | Valid Loss=2.5204 | Train Loss Unweighted=0.7391 | Valid Loss Unweighted=0.8354 | Stale=0 | ETA=2037.9m +rank0 [2026-11-15 06:51:35,393] INFO : split_configs=['1'] +rank0 [2026-11-15 06:51:35,414] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:51:35,422] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:51:35,531] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:51:36,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:51:36,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:51:37,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:51:37,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:51:37,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:51:38,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:51:38,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:51:38,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:51:39,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:51:39,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:51:39,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:51:40,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:51:40,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:51:40,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:51:41,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:51:41,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:51:41,702] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:51:41,705] INFO : split_configs=['1'] +rank0 [2026-11-15 06:51:41,716] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:51:41,719] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:51:41,795] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:51:42,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:51:43,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:51:43,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:51:44,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:51:44,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:51:45,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:51:45,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:51:46,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:51:46,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:51:46,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:51:47,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:51:47,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:51:48,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:51:48,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:51:49,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:51:49,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:51:49,647] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:51:49,651] INFO : split_configs=['1'] +rank0 [2026-11-15 06:51:49,662] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:51:49,665] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:51:49,742] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:51:50,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:51:51,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:51:51,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:51:52,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:51:52,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:51:52,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:51:53,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:51:53,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:51:54,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:51:54,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:51:54,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:51:55,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:51:55,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:51:56,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:51:56,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:51:57,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:51:57,114] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:51:57,140] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf +rank0 [2026-11-15 06:51:58,312] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_qq_pf +rank0 [2026-11-15 06:51:58,745] INFO : Plotted number of elements +rank0 [2026-11-15 06:52:00,025] INFO : Plotted jets +rank0 [2026-11-15 06:52:01,599] INFO : Plotted jet ratio +rank0 [2026-11-15 06:52:01,611] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:52:03,286] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:52:03,720] INFO : Plotted number of elements +rank0 [2026-11-15 06:52:05,078] INFO : Plotted jets +rank0 [2026-11-15 06:52:06,108] INFO : Plotted jet ratio +rank0 [2026-11-15 06:52:06,119] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:52:07,586] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_59000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:52:08,028] INFO : Plotted number of elements +rank0 [2026-11-15 06:52:09,363] INFO : Plotted jets +rank0 [2026-11-15 06:52:10,353] INFO : Plotted jet ratio +rank0 [2026-11-15 06:56:04,779] INFO : Step 60000/500000 rank0 | Train Loss: 1.8546 | Train Loss Unweighted: 0.7357 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2270 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2407 | Regression_eta: 0.0089 | Regression_pt: 0.2404 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1195 | Classification_binary: 0.2270 | Regression_cos_phi: 0.6235 | Regression_energy: 0.1440 | Regression_eta: 0.1153 | Regression_pt: 0.1438 | Regression_sin_phi: 0.4815 | LR: 1.86e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5127s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0550 | max_loss: 0.0028 | max_backward: 0.1164 | max_optimizer_step: 0.0892 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8148.9 | uss_MB: 7004.7 | child_rss_MB: 23923.1 | allocated_MB: 192.2 | reserved_MB: 20668.0 | inactive_split_MB: 151.8 +rank0 [2026-11-15 06:56:04,783] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-60000.pth +rank0 [2026-11-15 06:56:04,850] INFO : Running validation on rank0 +rank0 [2026-11-15 06:56:04,850] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 06:56:04,850] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 06:56:05,479] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-15 06:56:05,604] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 06:56:05,719] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 06:56:05,791] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 06:56:05,859] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 06:56:05,928] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 06:56:05,997] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 06:56:06,068] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 06:56:06,180] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 06:56:06,285] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 06:56:06,388] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 06:56:06,456] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 06:56:06,522] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 06:56:06,587] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 06:56:06,653] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 06:56:06,718] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 06:56:06,820] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 06:56:06,828] INFO : plotted pt_target histogram +rank0 [2026-11-15 06:56:06,831] INFO : plotted pt_pred histogram +rank0 [2026-11-15 06:56:06,833] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 06:56:06,836] INFO : plotted eta_target histogram +rank0 [2026-11-15 06:56:06,839] INFO : plotted eta_pred histogram +rank0 [2026-11-15 06:56:06,841] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 06:56:06,843] INFO : plotted sphi_target histogram +rank0 [2026-11-15 06:56:06,846] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 06:56:06,848] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 06:56:06,851] INFO : plotted cphi_target histogram +rank0 [2026-11-15 06:56:06,853] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 06:56:06,856] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 06:56:06,858] INFO : plotted energy_target histogram +rank0 [2026-11-15 06:56:06,861] INFO : plotted energy_pred histogram +rank0 [2026-11-15 06:56:06,863] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 06:56:14,550] INFO : VALIDATION | Step=60000/500000 | Train Loss=1.8546 | Valid Loss=2.6250 | Train Loss Unweighted=0.7357 | Valid Loss Unweighted=0.8436 | Stale=1 | ETA=2033.6m +rank0 [2026-11-15 06:56:14,550] INFO : split_configs=['1'] +rank0 [2026-11-15 06:56:14,562] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 06:56:14,568] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 06:56:14,683] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:56:15,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 06:56:15,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 06:56:16,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 06:56:16,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 06:56:17,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 06:56:17,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 06:56:17,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 06:56:18,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 06:56:18,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 06:56:18,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 06:56:19,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 06:56:19,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 06:56:19,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 06:56:20,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 06:56:20,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 06:56:20,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 06:56:20,842] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 06:56:20,845] INFO : split_configs=['1'] +rank0 [2026-11-15 06:56:20,856] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 06:56:20,859] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 06:56:20,932] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:56:22,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 06:56:22,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 06:56:22,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 06:56:23,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 06:56:23,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 06:56:24,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 06:56:24,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 06:56:25,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 06:56:25,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 06:56:26,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 06:56:26,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 06:56:26,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 06:56:27,414] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 06:56:27,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 06:56:28,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 06:56:28,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 06:56:28,834] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 06:56:28,838] INFO : split_configs=['1'] +rank0 [2026-11-15 06:56:28,849] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 06:56:28,853] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:56:28,927] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 06:56:30,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 06:56:30,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 06:56:30,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 06:56:31,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 06:56:31,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 06:56:32,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 06:56:32,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 06:56:32,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 06:56:33,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 06:56:33,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 06:56:34,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 06:56:34,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 06:56:34,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 06:56:35,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 06:56:35,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 06:56:36,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 06:56:36,352] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 06:56:36,361] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf +rank0 [2026-11-15 06:56:37,587] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_qq_pf +rank0 [2026-11-15 06:56:38,017] INFO : Plotted number of elements +rank0 [2026-11-15 06:56:39,302] INFO : Plotted jets +rank0 [2026-11-15 06:56:40,849] INFO : Plotted jet ratio +rank0 [2026-11-15 06:56:40,861] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:56:42,576] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ttbar_pf +rank0 [2026-11-15 06:56:43,005] INFO : Plotted number of elements +rank0 [2026-11-15 06:56:44,356] INFO : Plotted jets +rank0 [2026-11-15 06:56:45,380] INFO : Plotted jet ratio +rank0 [2026-11-15 06:56:45,391] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:56:46,881] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_60000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 06:56:47,326] INFO : Plotted number of elements +rank0 [2026-11-15 06:56:48,616] INFO : Plotted jets +rank0 [2026-11-15 06:56:49,601] INFO : Plotted jet ratio +rank0 [2026-11-15 06:56:50,060] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 06:56:50,060] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:00:43,595] INFO : Step 61000/500000 rank0 | Train Loss: 1.5491 | Train Loss Unweighted: 0.7963 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2433 | Regression_cos_phi: 0.0063 | Regression_energy: 0.2723 | Regression_eta: 0.0131 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.2433 | Regression_cos_phi: 0.3716 | Regression_energy: 0.1630 | Regression_eta: 0.1704 | Regression_pt: 0.1525 | Regression_sin_phi: 0.3810 | LR: 1.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5466s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0625 | max_loss: 0.0029 | max_backward: 0.1327 | max_optimizer_step: 0.0911 | max_tensorboard_logging: 0.8605 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 9900.2 | uss_MB: 8756.0 | child_rss_MB: 23680.6 | allocated_MB: 197.6 | reserved_MB: 21298.0 | inactive_split_MB: 188.4 +rank0 [2026-11-15 07:00:43,598] INFO : Running validation on rank0 +rank0 [2026-11-15 07:00:43,598] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:00:43,598] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:00:44,225] INFO : saved batch0_epoch61000.parquet +rank0 [2026-11-15 07:00:44,347] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:00:44,461] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:00:44,530] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:00:44,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:00:44,663] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:00:44,729] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:00:44,795] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:00:44,905] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:00:45,010] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:00:45,115] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:00:45,183] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:00:45,248] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:00:45,313] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:00:45,379] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:00:45,446] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:00:45,549] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:00:45,568] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:00:45,581] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:00:45,584] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:00:45,594] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:00:45,607] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:00:45,610] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:00:45,620] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:00:45,625] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:00:45,627] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:00:45,637] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:00:45,650] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:00:45,651] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:00:45,690] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:00:45,692] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:00:45,693] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:00:53,353] INFO : VALIDATION | Step=61000/500000 | Train Loss=1.5491 | Valid Loss=2.5733 | Train Loss Unweighted=0.7963 | Valid Loss Unweighted=0.8379 | Stale=2 | ETA=2029.1m +rank0 [2026-11-15 07:00:53,353] INFO : split_configs=['1'] +rank0 [2026-11-15 07:00:53,365] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:00:53,373] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:00:53,483] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:00:54,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:00:54,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:00:55,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:00:55,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:00:55,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:00:56,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:00:56,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:00:56,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:00:57,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:00:57,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:00:57,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:00:58,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:00:58,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:00:58,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:00:59,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:00:59,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:00:59,679] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:00:59,682] INFO : split_configs=['1'] +rank0 [2026-11-15 07:00:59,693] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:00:59,696] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:00:59,774] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:01:01,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:01:01,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:01:01,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:01:02,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:01:02,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:01:03,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:01:03,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:01:04,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:01:04,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:01:04,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:01:05,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:01:05,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:01:06,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:01:06,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:01:07,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:01:07,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:01:07,761] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:01:07,764] INFO : split_configs=['1'] +rank0 [2026-11-15 07:01:07,775] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:01:07,779] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:01:07,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:01:08,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:01:09,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:01:09,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:01:10,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:01:10,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:01:11,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:01:11,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:01:11,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:01:12,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:01:12,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:01:13,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:01:13,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:01:13,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:01:14,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:01:14,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:01:15,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:01:15,229] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:01:15,237] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf +rank0 [2026-11-15 07:01:16,428] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_qq_pf +rank0 [2026-11-15 07:01:16,843] INFO : Plotted number of elements +rank0 [2026-11-15 07:01:18,133] INFO : Plotted jets +rank0 [2026-11-15 07:01:19,683] INFO : Plotted jet ratio +rank0 [2026-11-15 07:01:19,693] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:01:21,383] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:01:21,811] INFO : Plotted number of elements +rank0 [2026-11-15 07:01:23,173] INFO : Plotted jets +rank0 [2026-11-15 07:01:24,206] INFO : Plotted jet ratio +rank0 [2026-11-15 07:01:24,214] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:01:25,739] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_61000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:01:26,188] INFO : Plotted number of elements +rank0 [2026-11-15 07:01:27,527] INFO : Plotted jets +rank0 [2026-11-15 07:01:28,514] INFO : Plotted jet ratio +rank0 [2026-11-15 07:03:25,087] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 07:03:25,088] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:05:22,067] INFO : Step 62000/500000 rank0 | Train Loss: 1.8277 | Train Loss Unweighted: 0.7590 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2553 | Regression_cos_phi: 0.0063 | Regression_energy: 0.2335 | Regression_eta: 0.0399 | Regression_pt: 0.2186 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0923 | Classification_binary: 0.2553 | Regression_cos_phi: 0.3714 | Regression_energy: 0.1398 | Regression_eta: 0.5179 | Regression_pt: 0.1307 | Regression_sin_phi: 0.3204 | LR: 1.95e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5212s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0631 | max_loss: 0.0028 | max_backward: 0.1336 | max_optimizer_step: 0.0921 | max_tensorboard_logging: 0.8407 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9894.2 | uss_MB: 8750.1 | child_rss_MB: 23770.0 | allocated_MB: 198.1 | reserved_MB: 19254.0 | inactive_split_MB: 361.9 +rank0 [2026-11-15 07:05:22,070] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-62000.pth +rank0 [2026-11-15 07:05:22,136] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 07:05:22,150] INFO : Running validation on rank0 +rank0 [2026-11-15 07:05:22,150] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:05:22,150] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:05:22,774] INFO : saved batch0_epoch62000.parquet +rank0 [2026-11-15 07:05:22,898] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:05:23,015] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:05:23,089] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:05:23,157] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:05:23,224] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:05:23,291] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:05:23,357] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:05:23,466] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:05:23,571] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:05:23,674] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:05:23,742] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:05:23,808] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:05:23,873] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:05:23,940] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:05:24,006] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:05:24,109] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:05:24,154] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:05:24,181] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:05:24,184] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:05:24,211] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:05:24,238] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:05:24,241] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:05:24,258] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:05:24,262] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:05:24,264] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:05:24,267] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:05:24,269] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:05:24,272] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:05:24,278] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:05:24,280] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:05:24,283] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:05:31,971] INFO : VALIDATION | Step=62000/500000 | Train Loss=1.8277 | Valid Loss=2.5907 | Train Loss Unweighted=0.7590 | Valid Loss Unweighted=0.8503 | Stale=3 | ETA=2024.6m +rank0 [2026-11-15 07:05:31,971] INFO : split_configs=['1'] +rank0 [2026-11-15 07:05:31,983] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:05:31,992] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:05:32,102] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:05:33,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:05:33,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:05:33,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:05:34,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:05:34,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:05:34,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:05:35,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:05:35,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:05:35,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:05:36,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:05:36,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:05:36,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:05:37,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:05:37,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:05:37,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:05:38,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:05:38,316] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:05:38,320] INFO : split_configs=['1'] +rank0 [2026-11-15 07:05:38,331] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:05:38,336] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:05:38,416] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:05:39,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:05:40,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:05:40,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:05:40,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:05:41,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:05:41,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:05:42,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:05:42,727] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:05:43,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:05:43,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:05:44,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:05:44,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:05:44,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:05:45,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:05:45,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:05:46,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:05:46,425] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:05:46,428] INFO : split_configs=['1'] +rank0 [2026-11-15 07:05:46,440] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:05:46,444] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:05:46,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:05:47,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:05:48,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:05:48,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:05:48,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:05:49,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:05:49,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:05:50,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:05:50,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:05:50,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:05:51,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:05:51,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:05:52,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:05:52,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:05:53,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:05:53,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:05:53,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:05:53,957] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:05:53,976] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf +rank0 [2026-11-15 07:05:55,193] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_qq_pf +rank0 [2026-11-15 07:05:55,623] INFO : Plotted number of elements +rank0 [2026-11-15 07:05:56,943] INFO : Plotted jets +rank0 [2026-11-15 07:05:58,526] INFO : Plotted jet ratio +rank0 [2026-11-15 07:05:58,550] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:06:00,202] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:06:00,637] INFO : Plotted number of elements +rank0 [2026-11-15 07:06:01,995] INFO : Plotted jets +rank0 [2026-11-15 07:06:03,027] INFO : Plotted jet ratio +rank0 [2026-11-15 07:06:03,048] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:06:04,511] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_62000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:06:04,952] INFO : Plotted number of elements +rank0 [2026-11-15 07:06:06,284] INFO : Plotted jets +rank0 [2026-11-15 07:06:07,276] INFO : Plotted jet ratio +rank0 [2026-11-15 07:10:01,921] INFO : Step 63000/500000 rank0 | Train Loss: 2.2889 | Train Loss Unweighted: 0.8857 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2466 | Regression_cos_phi: 0.0058 | Regression_energy: 0.2948 | Regression_eta: 0.0423 | Regression_pt: 0.2848 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1127 | Classification_binary: 0.2466 | Regression_cos_phi: 0.3427 | Regression_energy: 0.1764 | Regression_eta: 0.5494 | Regression_pt: 0.1703 | Regression_sin_phi: 0.6906 | LR: 2.00e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5475s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0577 | max_loss: 0.0029 | max_backward: 0.1231 | max_optimizer_step: 0.0922 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 8373.2 | uss_MB: 7229.0 | child_rss_MB: 23872.9 | allocated_MB: 193.2 | reserved_MB: 18586.0 | inactive_split_MB: 306.8 +rank0 [2026-11-15 07:10:01,923] INFO : Running validation on rank0 +rank0 [2026-11-15 07:10:01,924] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:10:01,924] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:10:02,561] INFO : saved batch0_epoch63000.parquet +rank0 [2026-11-15 07:10:02,688] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:10:02,805] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:10:02,876] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:10:02,944] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:10:03,012] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:10:03,084] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:10:03,155] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:10:03,268] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:10:03,378] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:10:03,485] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:10:03,556] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:10:03,624] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:10:03,691] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:10:03,758] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:10:03,825] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:10:03,930] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:10:03,937] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:10:03,940] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:10:03,943] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:10:03,946] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:10:03,949] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:10:03,951] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:10:03,953] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:10:03,956] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:10:03,958] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:10:03,961] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:10:03,963] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:10:03,966] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:10:03,968] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:10:03,971] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:10:03,973] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:10:11,732] INFO : VALIDATION | Step=63000/500000 | Train Loss=2.2889 | Valid Loss=2.6352 | Train Loss Unweighted=0.8857 | Valid Loss Unweighted=0.8513 | Stale=4 | ETA=2020.3m +rank0 [2026-11-15 07:10:11,732] INFO : split_configs=['1'] +rank0 [2026-11-15 07:10:11,744] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:10:11,751] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:10:11,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:10:12,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:10:13,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:10:13,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:10:13,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:10:14,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:10:14,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:10:14,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:10:15,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:10:15,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:10:15,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:10:16,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:10:16,647] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:10:16,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:10:17,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:10:17,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:10:17,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:10:18,069] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:10:18,073] INFO : split_configs=['1'] +rank0 [2026-11-15 07:10:18,085] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:10:18,088] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:10:18,159] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:10:19,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:10:19,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:10:20,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:10:20,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:10:21,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:10:21,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:10:21,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:10:22,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:10:22,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:10:23,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:10:23,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:10:24,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:10:24,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:10:25,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:10:25,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:10:25,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:10:26,086] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:10:26,090] INFO : split_configs=['1'] +rank0 [2026-11-15 07:10:26,101] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:10:26,105] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:10:26,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:10:27,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:10:27,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:10:28,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:10:28,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:10:28,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:10:29,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:10:29,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:10:30,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:10:30,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:10:30,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:10:31,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:10:31,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:10:32,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:10:32,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:10:33,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:10:33,479] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:10:33,587] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:10:33,595] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf +rank0 [2026-11-15 07:10:34,812] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_qq_pf +rank0 [2026-11-15 07:10:35,245] INFO : Plotted number of elements +rank0 [2026-11-15 07:10:36,550] INFO : Plotted jets +rank0 [2026-11-15 07:10:37,569] INFO : Plotted jet ratio +rank0 [2026-11-15 07:10:37,579] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:10:39,782] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:10:40,226] INFO : Plotted number of elements +rank0 [2026-11-15 07:10:41,614] INFO : Plotted jets +rank0 [2026-11-15 07:10:42,667] INFO : Plotted jet ratio +rank0 [2026-11-15 07:10:42,677] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:10:44,144] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_63000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:10:44,582] INFO : Plotted number of elements +rank0 [2026-11-15 07:10:45,913] INFO : Plotted jets +rank0 [2026-11-15 07:10:46,899] INFO : Plotted jet ratio +rank0 [2026-11-15 07:10:47,369] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 07:10:47,370] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:14:40,533] INFO : Step 64000/500000 rank0 | Train Loss: 1.4805 | Train Loss Unweighted: 0.8263 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2328 | Regression_cos_phi: 0.0055 | Regression_energy: 0.2774 | Regression_eta: 0.0147 | Regression_pt: 0.2908 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0925 | Classification_binary: 0.2328 | Regression_cos_phi: 0.3218 | Regression_energy: 0.1660 | Regression_eta: 0.1908 | Regression_pt: 0.1739 | Regression_sin_phi: 0.3026 | LR: 2.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5421s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0608 | max_loss: 0.0031 | max_backward: 0.1269 | max_optimizer_step: 0.0908 | max_tensorboard_logging: 0.8653 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9960.6 | uss_MB: 8816.4 | child_rss_MB: 23802.8 | allocated_MB: 198.8 | reserved_MB: 19994.0 | inactive_split_MB: 469.2 +rank0 [2026-11-15 07:14:40,536] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-64000.pth +rank0 [2026-11-15 07:14:40,602] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 07:14:40,616] INFO : Running validation on rank0 +rank0 [2026-11-15 07:14:40,616] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:14:40,616] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:14:41,240] INFO : saved batch0_epoch64000.parquet +rank0 [2026-11-15 07:14:41,362] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:14:41,479] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:14:41,551] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:14:41,621] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:14:41,688] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:14:41,756] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:14:41,823] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:14:41,935] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:14:42,041] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:14:42,148] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:14:42,218] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:14:42,287] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:14:42,356] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:14:42,425] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:14:42,493] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:14:42,598] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:14:42,641] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:14:42,645] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:14:42,646] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:14:42,647] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:14:42,648] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:14:42,649] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:14:42,651] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:14:42,652] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:14:42,661] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:14:42,662] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:14:42,663] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:14:42,664] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:14:42,665] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:14:42,667] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:14:42,668] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:14:50,414] INFO : VALIDATION | Step=64000/500000 | Train Loss=1.4805 | Valid Loss=2.6004 | Train Loss Unweighted=0.8263 | Valid Loss Unweighted=0.8299 | Stale=5 | ETA=2015.8m +rank0 [2026-11-15 07:14:50,415] INFO : split_configs=['1'] +rank0 [2026-11-15 07:14:50,426] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:14:50,434] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:14:50,545] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:14:51,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:14:51,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:14:52,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:14:52,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:14:52,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:14:53,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:14:53,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:14:53,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:14:54,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:14:54,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:14:55,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:14:55,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:14:55,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:14:55,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:14:56,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:14:56,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:14:56,796] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:14:56,799] INFO : split_configs=['1'] +rank0 [2026-11-15 07:14:56,811] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:14:56,814] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:14:56,892] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:14:58,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:14:58,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:14:58,967] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:14:59,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:14:59,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:15:00,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:15:00,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:15:01,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:15:01,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:15:02,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:15:02,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:15:02,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:15:03,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:15:03,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:15:04,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:15:04,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:15:04,845] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:15:04,849] INFO : split_configs=['1'] +rank0 [2026-11-15 07:15:04,861] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:15:04,865] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:15:04,947] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:15:06,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:15:06,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:15:06,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:15:07,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:15:07,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:15:08,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:15:08,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:15:08,955] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:15:09,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:15:09,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:15:10,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:15:10,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:15:11,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:15:11,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:15:11,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:15:12,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:15:12,352] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:15:12,359] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf +rank0 [2026-11-15 07:15:13,554] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_qq_pf +rank0 [2026-11-15 07:15:13,972] INFO : Plotted number of elements +rank0 [2026-11-15 07:15:15,269] INFO : Plotted jets +rank0 [2026-11-15 07:15:16,851] INFO : Plotted jet ratio +rank0 [2026-11-15 07:15:16,862] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:15:18,518] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:15:18,946] INFO : Plotted number of elements +rank0 [2026-11-15 07:15:20,330] INFO : Plotted jets +rank0 [2026-11-15 07:15:21,352] INFO : Plotted jet ratio +rank0 [2026-11-15 07:15:21,363] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:15:22,865] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_64000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:15:23,303] INFO : Plotted number of elements +rank0 [2026-11-15 07:15:24,633] INFO : Plotted jets +rank0 [2026-11-15 07:15:25,616] INFO : Plotted jet ratio +rank0 [2026-11-15 07:17:22,499] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 07:17:22,499] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:19:20,586] INFO : Step 65000/500000 rank0 | Train Loss: 1.9476 | Train Loss Unweighted: 0.8759 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0081 | Regression_energy: 0.3007 | Regression_eta: 0.0351 | Regression_pt: 0.2962 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0912 | Classification_binary: 0.2301 | Regression_cos_phi: 0.4734 | Regression_energy: 0.1799 | Regression_eta: 0.4564 | Regression_pt: 0.1772 | Regression_sin_phi: 0.3394 | LR: 2.10e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7422s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0023 | max_forward: 0.0605 | max_loss: 0.0030 | max_backward: 0.1287 | max_optimizer_step: 0.0914 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10036.1 | uss_MB: 8891.9 | child_rss_MB: 23884.6 | allocated_MB: 195.2 | reserved_MB: 22066.0 | inactive_split_MB: 294.8 +rank0 [2026-11-15 07:19:20,590] INFO : Running validation on rank0 +rank0 [2026-11-15 07:19:20,590] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:19:20,590] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:19:21,220] INFO : saved batch0_epoch65000.parquet +rank0 [2026-11-15 07:19:21,341] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:19:21,455] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:19:21,525] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:19:21,591] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:19:21,657] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:19:21,724] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:19:21,791] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:19:21,901] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:19:22,005] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:19:22,110] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:19:22,179] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:19:22,246] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:19:22,312] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:19:22,380] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:19:22,448] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:19:22,553] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:19:22,573] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:19:22,585] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:19:22,588] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:19:22,610] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:19:22,616] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:19:22,618] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:19:22,621] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:19:22,630] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:19:22,632] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:19:22,635] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:19:22,637] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:19:22,640] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:19:22,642] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:19:22,645] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:19:22,647] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:19:30,363] INFO : VALIDATION | Step=65000/500000 | Train Loss=1.9476 | Valid Loss=2.5314 | Train Loss Unweighted=0.8759 | Valid Loss Unweighted=0.8316 | Stale=6 | ETA=2011.5m +rank0 [2026-11-15 07:19:30,363] INFO : split_configs=['1'] +rank0 [2026-11-15 07:19:30,375] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:19:30,383] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:19:30,488] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:19:31,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:19:31,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:19:32,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:19:32,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:19:32,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:19:33,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:19:33,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:19:33,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:19:34,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:19:34,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:19:34,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:19:35,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:19:35,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:19:35,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:19:36,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:19:36,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:19:36,642] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:19:36,646] INFO : split_configs=['1'] +rank0 [2026-11-15 07:19:36,658] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:19:36,661] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:19:36,733] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:19:37,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:19:38,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:19:38,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:19:39,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:19:39,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:19:40,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:19:40,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:19:40,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:19:41,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:19:41,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:19:42,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:19:42,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:19:43,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:19:43,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:19:44,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:19:44,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:19:44,604] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:19:44,607] INFO : split_configs=['1'] +rank0 [2026-11-15 07:19:44,621] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:19:44,624] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:19:44,698] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:19:45,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:19:46,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:19:46,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:19:47,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:19:47,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:19:47,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:19:48,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:19:48,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:19:49,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:19:49,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:19:49,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:19:50,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:19:50,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:19:51,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:19:51,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:19:51,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:19:52,094] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:19:52,102] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf +rank0 [2026-11-15 07:19:53,317] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_qq_pf +rank0 [2026-11-15 07:19:53,752] INFO : Plotted number of elements +rank0 [2026-11-15 07:19:55,065] INFO : Plotted jets +rank0 [2026-11-15 07:19:56,094] INFO : Plotted jet ratio +rank0 [2026-11-15 07:19:56,105] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:19:57,797] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:19:58,785] INFO : Plotted number of elements +rank0 [2026-11-15 07:20:00,161] INFO : Plotted jets +rank0 [2026-11-15 07:20:01,229] INFO : Plotted jet ratio +rank0 [2026-11-15 07:20:01,239] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:20:02,715] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_65000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:20:03,161] INFO : Plotted number of elements +rank0 [2026-11-15 07:20:04,500] INFO : Plotted jets +rank0 [2026-11-15 07:20:05,501] INFO : Plotted jet ratio +rank0 [2026-11-15 07:23:59,569] INFO : Step 66000/500000 rank0 | Train Loss: 1.9954 | Train Loss Unweighted: 0.8365 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2530 | Regression_cos_phi: 0.0072 | Regression_energy: 0.2808 | Regression_eta: 0.0132 | Regression_pt: 0.2706 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0982 | Classification_binary: 0.2530 | Regression_cos_phi: 0.4205 | Regression_energy: 0.1680 | Regression_eta: 0.1713 | Regression_pt: 0.1618 | Regression_sin_phi: 0.7226 | LR: 2.15e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.7803s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0614 | max_loss: 0.0031 | max_backward: 0.1299 | max_optimizer_step: 0.0922 | max_tensorboard_logging: 1.0998 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10090.1 | uss_MB: 8946.0 | child_rss_MB: 23639.5 | allocated_MB: 196.8 | reserved_MB: 15542.0 | inactive_split_MB: 209.2 +rank0 [2026-11-15 07:23:59,573] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-66000.pth +rank0 [2026-11-15 07:23:59,651] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 07:23:59,665] INFO : Running validation on rank0 +rank0 [2026-11-15 07:23:59,665] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:23:59,665] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:24:00,291] INFO : saved batch0_epoch66000.parquet +rank0 [2026-11-15 07:24:00,412] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:24:00,530] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:24:00,602] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:24:00,670] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:24:00,737] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:24:00,804] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:24:00,872] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:24:00,981] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:24:01,086] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:24:01,189] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:24:01,258] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:24:01,325] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:24:01,391] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:24:01,458] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:24:01,524] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:24:01,626] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:24:01,634] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:24:01,637] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:24:01,639] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:24:01,642] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:24:01,644] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:24:01,647] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:24:01,649] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:24:01,652] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:24:01,654] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:24:01,656] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:24:01,659] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:24:01,661] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:24:01,676] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:24:01,679] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:24:01,681] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:24:09,413] INFO : VALIDATION | Step=66000/500000 | Train Loss=1.9954 | Valid Loss=2.6272 | Train Loss Unweighted=0.8365 | Valid Loss Unweighted=0.8444 | Stale=7 | ETA=2007.0m +rank0 [2026-11-15 07:24:09,414] INFO : split_configs=['1'] +rank0 [2026-11-15 07:24:09,424] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:24:09,431] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:24:09,539] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:24:10,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:24:10,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:24:11,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:24:11,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:24:11,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:24:12,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:24:12,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:24:12,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:24:13,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:24:13,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:24:14,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:24:14,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:24:14,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:24:15,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:24:15,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:24:15,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:24:15,825] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:24:15,828] INFO : split_configs=['1'] +rank0 [2026-11-15 07:24:15,838] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:24:15,842] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:24:15,922] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:24:17,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:24:17,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:24:18,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:24:18,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:24:18,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:24:19,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:24:19,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:24:20,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:24:20,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:24:21,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:24:21,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:24:22,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:24:22,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:24:22,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:24:23,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:24:23,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:24:23,916] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:24:23,919] INFO : split_configs=['1'] +rank0 [2026-11-15 07:24:23,932] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:24:23,936] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:24:24,017] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:24:25,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:24:25,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:24:26,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:24:26,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:24:26,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:24:27,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:24:27,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:24:28,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:24:28,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:24:28,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:24:29,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:24:29,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:24:30,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:24:30,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:24:30,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:24:31,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:24:31,486] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:24:31,496] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf +rank0 [2026-11-15 07:24:32,713] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_qq_pf +rank0 [2026-11-15 07:24:33,130] INFO : Plotted number of elements +rank0 [2026-11-15 07:24:34,409] INFO : Plotted jets +rank0 [2026-11-15 07:24:35,425] INFO : Plotted jet ratio +rank0 [2026-11-15 07:24:35,436] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:24:37,652] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:24:38,081] INFO : Plotted number of elements +rank0 [2026-11-15 07:24:39,431] INFO : Plotted jets +rank0 [2026-11-15 07:24:40,463] INFO : Plotted jet ratio +rank0 [2026-11-15 07:24:40,472] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:24:41,906] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_66000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:24:42,369] INFO : Plotted number of elements +rank0 [2026-11-15 07:24:43,695] INFO : Plotted jets +rank0 [2026-11-15 07:24:44,689] INFO : Plotted jet ratio +rank0 [2026-11-15 07:24:45,167] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 07:24:45,167] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:28:38,162] INFO : Step 67000/500000 rank0 | Train Loss: 1.1077 | Train Loss Unweighted: 0.6002 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1884 | Regression_eta: 0.0119 | Regression_pt: 0.1817 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2104 | Regression_cos_phi: 0.2010 | Regression_energy: 0.1128 | Regression_eta: 0.1544 | Regression_pt: 0.1087 | Regression_sin_phi: 0.2578 | LR: 2.20e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6809s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0700 | max_loss: 0.0032 | max_backward: 0.1528 | max_optimizer_step: 0.0914 | max_tensorboard_logging: 0.9718 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10182.7 | uss_MB: 9038.5 | child_rss_MB: 23809.8 | allocated_MB: 198.4 | reserved_MB: 21646.0 | inactive_split_MB: 381.6 +rank0 [2026-11-15 07:28:38,165] INFO : Running validation on rank0 +rank0 [2026-11-15 07:28:38,165] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:28:38,165] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:28:38,793] INFO : saved batch0_epoch67000.parquet +rank0 [2026-11-15 07:28:38,914] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:28:39,029] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:28:39,101] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:28:39,170] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:28:39,239] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:28:39,309] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:28:39,380] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:28:39,493] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:28:39,600] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:28:39,703] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:28:39,771] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:28:39,839] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:28:39,909] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:28:39,975] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:28:40,042] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:28:40,148] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:28:40,155] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:28:40,158] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:28:40,160] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:28:40,163] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:28:40,166] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:28:40,168] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:28:40,171] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:28:40,173] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:28:40,176] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:28:40,178] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:28:40,181] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:28:40,183] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:28:40,185] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:28:40,188] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:28:40,190] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:28:47,891] INFO : VALIDATION | Step=67000/500000 | Train Loss=1.1077 | Valid Loss=2.5452 | Train Loss Unweighted=0.6002 | Valid Loss Unweighted=0.8299 | Stale=8 | ETA=2002.5m +rank0 [2026-11-15 07:28:47,892] INFO : split_configs=['1'] +rank0 [2026-11-15 07:28:47,903] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:28:47,909] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:28:48,045] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:28:49,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:28:49,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:28:49,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:28:50,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:28:50,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:28:50,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:28:51,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:28:51,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:28:51,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:28:52,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:28:52,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:28:52,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:28:53,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:28:53,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:28:53,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:28:54,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:28:54,249] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:28:54,253] INFO : split_configs=['1'] +rank0 [2026-11-15 07:28:54,264] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:28:54,267] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:28:54,347] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:28:55,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:28:55,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:28:56,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:28:56,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:28:57,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:28:57,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:28:58,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:28:58,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:28:59,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:28:59,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:28:59,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:29:00,382] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:29:00,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:29:01,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:29:01,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:29:02,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:29:02,261] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:29:02,264] INFO : split_configs=['1'] +rank0 [2026-11-15 07:29:02,275] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:29:02,279] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:29:02,359] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:29:03,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:29:03,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:29:04,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:29:04,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:29:05,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:29:05,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:29:05,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:29:06,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:29:06,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:29:07,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:29:07,555] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:29:07,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:29:08,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:29:08,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:29:09,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:29:09,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:29:09,727] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:29:09,736] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf +rank0 [2026-11-15 07:29:10,949] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_qq_pf +rank0 [2026-11-15 07:29:11,380] INFO : Plotted number of elements +rank0 [2026-11-15 07:29:12,652] INFO : Plotted jets +rank0 [2026-11-15 07:29:13,679] INFO : Plotted jet ratio +rank0 [2026-11-15 07:29:13,689] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:29:15,846] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:29:16,297] INFO : Plotted number of elements +rank0 [2026-11-15 07:29:17,630] INFO : Plotted jets +rank0 [2026-11-15 07:29:18,661] INFO : Plotted jet ratio +rank0 [2026-11-15 07:29:18,670] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:29:20,084] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_67000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:29:20,529] INFO : Plotted number of elements +rank0 [2026-11-15 07:29:21,845] INFO : Plotted jets +rank0 [2026-11-15 07:29:22,843] INFO : Plotted jet ratio +rank0 [2026-11-15 07:31:19,867] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 07:31:19,867] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:33:17,439] INFO : Step 68000/500000 rank0 | Train Loss: 1.7513 | Train Loss Unweighted: 0.7508 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2397 | Regression_cos_phi: 0.0050 | Regression_energy: 0.2416 | Regression_eta: 0.0279 | Regression_pt: 0.2287 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0950 | Classification_binary: 0.2397 | Regression_cos_phi: 0.2918 | Regression_energy: 0.1446 | Regression_eta: 0.3625 | Regression_pt: 0.1368 | Regression_sin_phi: 0.4810 | LR: 2.25e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4313s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0027 | max_forward: 0.0594 | max_loss: 0.0029 | max_backward: 0.1252 | max_optimizer_step: 0.0917 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10196.1 | uss_MB: 9051.9 | child_rss_MB: 23717.6 | allocated_MB: 197.7 | reserved_MB: 18800.0 | inactive_split_MB: 190.3 +rank0 [2026-11-15 07:33:17,441] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-68000.pth +rank0 [2026-11-15 07:33:17,519] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 07:33:17,533] INFO : Running validation on rank0 +rank0 [2026-11-15 07:33:17,534] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:33:17,534] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:33:18,163] INFO : saved batch0_epoch68000.parquet +rank0 [2026-11-15 07:33:18,287] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:33:18,401] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:33:18,471] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:33:18,539] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:33:18,645] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:33:18,714] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:33:18,781] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:33:18,898] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:33:19,002] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:33:19,109] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:33:19,178] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:33:19,245] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:33:19,311] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:33:19,379] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:33:19,447] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:33:19,553] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:33:19,573] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:33:19,586] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:33:19,590] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:33:19,599] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:33:19,612] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:33:19,615] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:33:19,625] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:33:19,633] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:33:19,635] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:33:19,638] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:33:19,640] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:33:19,643] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:33:19,650] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:33:19,653] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:33:19,655] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:33:27,339] INFO : VALIDATION | Step=68000/500000 | Train Loss=1.7513 | Valid Loss=2.5741 | Train Loss Unweighted=0.7508 | Valid Loss Unweighted=0.8457 | Stale=9 | ETA=1998.1m +rank0 [2026-11-15 07:33:27,339] INFO : split_configs=['1'] +rank0 [2026-11-15 07:33:27,350] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:33:27,358] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:33:27,462] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:33:28,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:33:28,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:33:29,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:33:29,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:33:29,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:33:30,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:33:30,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:33:30,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:33:31,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:33:31,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:33:31,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:33:32,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:33:32,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:33:32,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:33:33,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:33:33,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:33:33,627] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:33:33,630] INFO : split_configs=['1'] +rank0 [2026-11-15 07:33:33,642] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:33:33,645] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:33:33,721] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:33:34,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:33:35,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:33:35,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:33:36,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:33:36,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:33:37,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:33:37,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:33:37,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:33:38,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:33:38,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:33:39,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:33:39,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:33:40,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:33:40,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:33:41,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:33:41,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:33:41,632] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:33:41,636] INFO : split_configs=['1'] +rank0 [2026-11-15 07:33:41,647] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:33:41,650] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:33:41,727] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:33:42,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:33:43,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:33:43,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:33:44,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:33:44,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:33:44,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:33:45,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:33:45,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:33:46,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:33:46,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:33:46,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:33:47,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:33:47,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:33:48,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:33:48,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:33:49,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:33:49,178] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:33:49,187] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf +rank0 [2026-11-15 07:33:50,418] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_qq_pf +rank0 [2026-11-15 07:33:50,852] INFO : Plotted number of elements +rank0 [2026-11-15 07:33:52,175] INFO : Plotted jets +rank0 [2026-11-15 07:33:53,200] INFO : Plotted jet ratio +rank0 [2026-11-15 07:33:53,211] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:33:54,874] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:33:55,862] INFO : Plotted number of elements +rank0 [2026-11-15 07:33:57,215] INFO : Plotted jets +rank0 [2026-11-15 07:33:58,251] INFO : Plotted jet ratio +rank0 [2026-11-15 07:33:58,261] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:33:59,718] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_68000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:34:00,158] INFO : Plotted number of elements +rank0 [2026-11-15 07:34:01,482] INFO : Plotted jets +rank0 [2026-11-15 07:34:02,467] INFO : Plotted jet ratio +rank0 [2026-11-15 07:37:56,492] INFO : Step 69000/500000 rank0 | Train Loss: 1.5408 | Train Loss Unweighted: 0.7317 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2385 | Regression_cos_phi: 0.0057 | Regression_energy: 0.2265 | Regression_eta: 0.0165 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2385 | Regression_cos_phi: 0.3342 | Regression_energy: 0.1355 | Regression_eta: 0.2144 | Regression_pt: 0.1423 | Regression_sin_phi: 0.3923 | LR: 2.30e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4574s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0024 | max_forward: 0.0651 | max_loss: 0.0028 | max_backward: 0.1414 | max_optimizer_step: 0.0921 | max_tensorboard_logging: 0.7611 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10232.5 | uss_MB: 9088.4 | child_rss_MB: 23794.0 | allocated_MB: 193.3 | reserved_MB: 18394.0 | inactive_split_MB: 268.7 +rank0 [2026-11-15 07:37:56,495] INFO : Running validation on rank0 +rank0 [2026-11-15 07:37:56,495] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:37:56,495] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:37:57,126] INFO : saved batch0_epoch69000.parquet +rank0 [2026-11-15 07:37:57,251] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:37:57,370] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:37:57,443] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:37:57,513] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:37:57,580] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:37:57,649] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:37:57,716] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:37:57,827] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:37:57,932] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:37:58,036] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:37:58,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:37:58,176] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:37:58,242] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:37:58,308] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:37:58,376] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:37:58,480] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:37:58,488] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:37:58,491] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:37:58,493] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:37:58,496] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:37:58,499] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:37:58,501] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:37:58,504] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:37:58,519] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:37:58,522] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:37:58,530] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:37:58,533] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:37:58,535] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:37:58,538] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:37:58,541] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:37:58,543] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:38:06,268] INFO : VALIDATION | Step=69000/500000 | Train Loss=1.5408 | Valid Loss=2.5331 | Train Loss Unweighted=0.7317 | Valid Loss Unweighted=0.8283 | Stale=10 | ETA=1993.6m +rank0 [2026-11-15 07:38:06,268] INFO : split_configs=['1'] +rank0 [2026-11-15 07:38:06,279] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:38:06,286] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:38:06,397] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:38:07,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:38:07,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:38:08,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:38:08,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:38:08,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:38:09,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:38:09,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:38:09,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:38:10,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:38:10,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:38:10,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:38:11,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:38:11,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:38:11,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:38:12,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:38:12,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:38:12,684] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:38:12,688] INFO : split_configs=['1'] +rank0 [2026-11-15 07:38:12,703] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:38:12,706] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:38:12,788] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:38:14,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:38:14,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:38:14,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:38:15,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:38:15,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:38:16,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:38:16,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:38:17,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:38:17,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:38:18,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:38:18,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:38:19,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:38:19,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:38:19,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:38:20,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:38:20,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:38:20,917] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:38:20,922] INFO : split_configs=['1'] +rank0 [2026-11-15 07:38:20,938] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:38:20,941] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:38:21,025] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:38:22,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:38:22,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:38:23,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:38:23,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:38:23,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:38:24,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:38:24,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:38:25,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:38:25,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:38:25,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:38:26,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:38:26,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:38:27,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:38:27,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:38:28,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:38:28,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:38:28,549] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:38:28,558] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf +rank0 [2026-11-15 07:38:29,730] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_qq_pf +rank0 [2026-11-15 07:38:30,155] INFO : Plotted number of elements +rank0 [2026-11-15 07:38:31,460] INFO : Plotted jets +rank0 [2026-11-15 07:38:32,487] INFO : Plotted jet ratio +rank0 [2026-11-15 07:38:32,497] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:38:34,102] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:38:35,098] INFO : Plotted number of elements +rank0 [2026-11-15 07:38:36,461] INFO : Plotted jets +rank0 [2026-11-15 07:38:37,516] INFO : Plotted jet ratio +rank0 [2026-11-15 07:38:37,525] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:38:38,981] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_69000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:38:39,445] INFO : Plotted number of elements +rank0 [2026-11-15 07:38:40,785] INFO : Plotted jets +rank0 [2026-11-15 07:38:41,814] INFO : Plotted jet ratio +rank0 [2026-11-15 07:38:42,313] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 07:38:42,313] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:42:35,949] INFO : Step 70000/500000 rank0 | Train Loss: 1.4311 | Train Loss Unweighted: 0.7400 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2326 | Regression_cos_phi: 0.0055 | Regression_energy: 0.2413 | Regression_eta: 0.0182 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0819 | Classification_binary: 0.2326 | Regression_cos_phi: 0.3227 | Regression_energy: 0.1444 | Regression_eta: 0.2358 | Regression_pt: 0.1422 | Regression_sin_phi: 0.2714 | LR: 2.35e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4952s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0026 | max_forward: 0.0659 | max_loss: 0.0029 | max_backward: 0.1421 | max_optimizer_step: 0.1024 | max_tensorboard_logging: 0.7785 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10323.3 | uss_MB: 9179.2 | child_rss_MB: 23859.9 | allocated_MB: 194.6 | reserved_MB: 17728.0 | inactive_split_MB: 273.4 +rank0 [2026-11-15 07:42:35,951] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-70000.pth +rank0 [2026-11-15 07:42:36,017] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 07:42:36,030] INFO : Running validation on rank0 +rank0 [2026-11-15 07:42:36,030] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:42:36,030] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:42:36,654] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-15 07:42:36,775] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:42:36,889] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:42:36,960] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:42:37,028] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:42:37,096] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:42:37,163] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:42:37,229] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:42:37,339] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:42:37,443] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:42:37,545] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:42:37,614] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:42:37,681] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:42:37,747] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:42:37,816] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:42:37,884] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:42:37,990] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:42:38,010] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:42:38,023] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:42:38,032] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:42:38,035] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:42:38,038] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:42:38,040] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:42:38,049] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:42:38,051] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:42:38,054] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:42:38,057] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:42:38,059] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:42:38,062] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:42:38,064] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:42:38,067] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:42:38,069] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:42:45,739] INFO : VALIDATION | Step=70000/500000 | Train Loss=1.4311 | Valid Loss=2.5769 | Train Loss Unweighted=0.7400 | Valid Loss Unweighted=0.8331 | Stale=11 | ETA=1989.2m +rank0 [2026-11-15 07:42:45,740] INFO : split_configs=['1'] +rank0 [2026-11-15 07:42:45,750] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:42:45,757] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:42:45,868] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:42:46,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:42:47,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:42:47,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:42:47,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:42:48,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:42:48,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:42:48,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:42:49,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:42:49,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:42:50,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:42:50,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:42:50,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:42:51,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:42:51,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:42:51,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:42:52,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:42:52,129] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:42:52,132] INFO : split_configs=['1'] +rank0 [2026-11-15 07:42:52,143] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:42:52,147] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:42:52,230] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:42:53,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:42:53,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:42:54,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:42:54,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:42:55,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:42:55,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:42:56,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:42:56,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:42:57,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:42:57,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:42:57,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:42:58,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:42:58,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:42:59,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:42:59,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:43:00,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:43:00,271] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:43:00,274] INFO : split_configs=['1'] +rank0 [2026-11-15 07:43:00,285] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:43:00,288] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:43:00,370] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:43:01,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:43:01,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:43:02,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:43:02,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:43:03,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:43:03,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:43:03,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:43:04,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:43:04,794] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:43:05,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:43:05,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:43:06,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:43:06,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:43:06,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:43:07,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:43:07,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:43:07,855] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:43:07,863] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf +rank0 [2026-11-15 07:43:09,060] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_qq_pf +rank0 [2026-11-15 07:43:09,482] INFO : Plotted number of elements +rank0 [2026-11-15 07:43:10,778] INFO : Plotted jets +rank0 [2026-11-15 07:43:11,799] INFO : Plotted jet ratio +rank0 [2026-11-15 07:43:11,808] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:43:13,459] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:43:13,899] INFO : Plotted number of elements +rank0 [2026-11-15 07:43:15,800] INFO : Plotted jets +rank0 [2026-11-15 07:43:16,839] INFO : Plotted jet ratio +rank0 [2026-11-15 07:43:16,848] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:43:18,322] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_70000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:43:18,761] INFO : Plotted number of elements +rank0 [2026-11-15 07:43:20,101] INFO : Plotted jets +rank0 [2026-11-15 07:43:21,092] INFO : Plotted jet ratio +rank0 [2026-11-15 07:45:17,384] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 07:45:17,384] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:47:14,796] INFO : Step 71000/500000 rank0 | Train Loss: 1.3493 | Train Loss Unweighted: 0.8246 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2303 | Regression_cos_phi: 0.0050 | Regression_energy: 0.2904 | Regression_eta: 0.0100 | Regression_pt: 0.2846 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2303 | Regression_cos_phi: 0.2957 | Regression_energy: 0.1738 | Regression_eta: 0.1295 | Regression_pt: 0.1702 | Regression_sin_phi: 0.2518 | LR: 2.40e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5103s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0681 | max_loss: 0.0028 | max_backward: 0.1440 | max_optimizer_step: 0.0920 | max_tensorboard_logging: 0.7528 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10415.9 | uss_MB: 9271.7 | child_rss_MB: 23714.8 | allocated_MB: 199.1 | reserved_MB: 20906.0 | inactive_split_MB: 120.9 +rank0 [2026-11-15 07:47:14,801] INFO : Running validation on rank0 +rank0 [2026-11-15 07:47:14,801] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:47:14,801] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:47:15,433] INFO : saved batch0_epoch71000.parquet +rank0 [2026-11-15 07:47:15,561] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:47:15,676] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:47:15,747] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:47:15,814] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:47:15,880] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:47:15,947] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:47:16,014] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:47:16,129] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:47:16,235] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:47:16,343] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:47:16,412] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:47:16,481] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:47:16,551] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:47:16,619] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:47:16,687] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:47:16,792] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:47:16,800] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:47:16,803] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:47:16,805] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:47:16,808] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:47:16,810] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:47:16,813] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:47:16,815] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:47:16,818] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:47:16,820] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:47:16,823] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:47:16,825] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:47:16,828] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:47:16,830] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:47:16,833] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:47:16,835] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:47:24,555] INFO : VALIDATION | Step=71000/500000 | Train Loss=1.3493 | Valid Loss=2.5839 | Train Loss Unweighted=0.8246 | Valid Loss Unweighted=0.8257 | Stale=12 | ETA=1984.7m +rank0 [2026-11-15 07:47:24,555] INFO : split_configs=['1'] +rank0 [2026-11-15 07:47:24,566] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:47:24,573] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:47:24,679] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:47:25,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:47:26,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:47:26,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:47:26,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:47:27,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:47:27,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:47:27,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:47:28,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:47:28,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:47:28,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:47:29,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:47:29,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:47:29,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:47:30,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:47:30,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:47:30,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:47:30,850] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:47:30,853] INFO : split_configs=['1'] +rank0 [2026-11-15 07:47:30,864] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:47:30,867] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:47:30,943] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:47:32,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:47:32,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:47:32,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:47:33,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:47:33,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:47:34,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:47:34,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:47:35,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:47:35,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:47:36,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:47:36,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:47:36,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:47:37,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:47:37,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:47:38,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:47:38,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:47:38,860] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:47:38,864] INFO : split_configs=['1'] +rank0 [2026-11-15 07:47:38,898] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:47:38,902] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:47:38,980] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:47:40,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:47:40,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:47:40,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:47:41,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:47:41,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:47:42,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:47:42,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:47:42,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:47:43,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:47:43,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:47:44,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:47:44,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:47:44,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:47:45,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:47:45,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:47:46,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:47:46,305] INFO : Time taken to make predictions on device 0 is: 0.11 min +rank0 [2026-11-15 07:47:46,313] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf +rank0 [2026-11-15 07:47:47,512] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_qq_pf +rank0 [2026-11-15 07:47:47,938] INFO : Plotted number of elements +rank0 [2026-11-15 07:47:49,241] INFO : Plotted jets +rank0 [2026-11-15 07:47:50,259] INFO : Plotted jet ratio +rank0 [2026-11-15 07:47:50,270] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:47:51,927] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:47:52,367] INFO : Plotted number of elements +rank0 [2026-11-15 07:47:54,276] INFO : Plotted jets +rank0 [2026-11-15 07:47:55,307] INFO : Plotted jet ratio +rank0 [2026-11-15 07:47:55,316] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:47:56,760] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_71000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:47:57,199] INFO : Plotted number of elements +rank0 [2026-11-15 07:47:58,527] INFO : Plotted jets +rank0 [2026-11-15 07:47:59,515] INFO : Plotted jet ratio +rank0 [2026-11-15 07:51:53,276] INFO : Step 72000/500000 rank0 | Train Loss: 2.3092 | Train Loss Unweighted: 0.7606 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2382 | Regression_cos_phi: 0.0118 | Regression_energy: 0.2363 | Regression_eta: 0.0337 | Regression_pt: 0.2310 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2382 | Regression_cos_phi: 0.6925 | Regression_energy: 0.1414 | Regression_eta: 0.4375 | Regression_pt: 0.1382 | Regression_sin_phi: 0.5802 | LR: 2.45e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4759s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0026 | max_forward: 0.0698 | max_loss: 0.0030 | max_backward: 0.1533 | max_optimizer_step: 0.0928 | max_tensorboard_logging: 0.7604 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 10461.5 | uss_MB: 9317.4 | child_rss_MB: 23745.1 | allocated_MB: 197.4 | reserved_MB: 19668.0 | inactive_split_MB: 438.6 +rank0 [2026-11-15 07:51:53,279] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-72000.pth +rank0 [2026-11-15 07:51:53,345] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 07:51:53,357] INFO : Running validation on rank0 +rank0 [2026-11-15 07:51:53,358] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:51:53,358] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:51:53,984] INFO : saved batch0_epoch72000.parquet +rank0 [2026-11-15 07:51:54,106] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:51:54,220] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:51:54,290] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:51:54,359] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:51:54,427] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:51:54,496] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:51:54,565] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:51:54,680] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:51:54,787] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:51:54,889] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:51:54,957] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:51:55,024] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:51:55,092] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:51:55,160] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:51:55,226] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:51:55,330] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:51:55,337] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:51:55,340] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:51:55,343] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:51:55,345] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:51:55,348] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:51:55,350] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:51:55,353] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:51:55,356] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:51:55,358] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:51:55,360] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:51:55,363] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:51:55,365] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:51:55,368] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:51:55,370] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:51:55,373] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:52:03,071] INFO : VALIDATION | Step=72000/500000 | Train Loss=2.3092 | Valid Loss=2.5534 | Train Loss Unweighted=0.7606 | Valid Loss Unweighted=0.8394 | Stale=13 | ETA=1980.2m +rank0 [2026-11-15 07:52:03,072] INFO : split_configs=['1'] +rank0 [2026-11-15 07:52:03,083] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:52:03,091] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:52:03,229] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:52:04,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:52:04,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:52:04,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:52:05,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:52:05,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:52:06,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:52:06,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:52:06,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:52:07,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:52:07,394] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:52:07,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:52:08,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:52:08,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:52:08,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:52:09,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:52:09,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:52:09,518] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:52:09,521] INFO : split_configs=['1'] +rank0 [2026-11-15 07:52:09,532] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:52:09,536] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:52:09,617] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:52:10,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:52:11,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:52:11,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:52:12,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:52:12,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:52:13,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:52:13,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:52:13,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:52:14,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:52:14,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:52:15,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:52:15,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:52:16,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:52:16,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:52:17,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:52:17,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:52:17,658] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:52:17,662] INFO : split_configs=['1'] +rank0 [2026-11-15 07:52:17,672] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:52:17,676] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:52:17,759] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:52:18,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:52:19,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:52:19,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:52:20,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:52:20,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:52:21,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:52:21,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:52:21,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:52:22,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:52:22,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:52:23,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:52:23,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:52:23,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:52:24,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:52:24,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:52:25,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:52:25,208] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:52:25,215] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf +rank0 [2026-11-15 07:52:26,402] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_qq_pf +rank0 [2026-11-15 07:52:26,827] INFO : Plotted number of elements +rank0 [2026-11-15 07:52:28,110] INFO : Plotted jets +rank0 [2026-11-15 07:52:29,129] INFO : Plotted jet ratio +rank0 [2026-11-15 07:52:29,140] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:52:30,833] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:52:31,825] INFO : Plotted number of elements +rank0 [2026-11-15 07:52:33,156] INFO : Plotted jets +rank0 [2026-11-15 07:52:34,187] INFO : Plotted jet ratio +rank0 [2026-11-15 07:52:34,196] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:52:35,650] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_72000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:52:36,116] INFO : Plotted number of elements +rank0 [2026-11-15 07:52:37,448] INFO : Plotted jets +rank0 [2026-11-15 07:52:38,462] INFO : Plotted jet ratio +rank0 [2026-11-15 07:52:38,941] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 07:52:38,941] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 07:56:31,593] INFO : Step 73000/500000 rank0 | Train Loss: 2.1618 | Train Loss Unweighted: 0.7444 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2273 | Regression_cos_phi: 0.0087 | Regression_energy: 0.2377 | Regression_eta: 0.0379 | Regression_pt: 0.2238 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1125 | Classification_binary: 0.2273 | Regression_cos_phi: 0.5097 | Regression_energy: 0.1422 | Regression_eta: 0.4926 | Regression_pt: 0.1339 | Regression_sin_phi: 0.5436 | LR: 2.50e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4289s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0024 | max_forward: 0.0548 | max_loss: 0.0029 | max_backward: 0.1157 | max_optimizer_step: 0.0946 | max_tensorboard_logging: 0.7384 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10537.4 | uss_MB: 9393.3 | child_rss_MB: 23841.6 | allocated_MB: 196.9 | reserved_MB: 17826.0 | inactive_split_MB: 337.1 +rank0 [2026-11-15 07:56:31,596] INFO : Running validation on rank0 +rank0 [2026-11-15 07:56:31,597] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 07:56:31,597] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 07:56:32,226] INFO : saved batch0_epoch73000.parquet +rank0 [2026-11-15 07:56:32,347] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 07:56:32,463] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 07:56:32,533] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 07:56:32,600] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 07:56:32,666] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 07:56:32,732] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 07:56:32,800] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 07:56:32,909] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 07:56:33,014] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 07:56:33,120] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 07:56:33,189] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 07:56:33,257] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 07:56:33,325] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 07:56:33,393] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 07:56:33,462] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 07:56:33,567] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 07:56:33,586] INFO : plotted pt_target histogram +rank0 [2026-11-15 07:56:33,599] INFO : plotted pt_pred histogram +rank0 [2026-11-15 07:56:33,602] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 07:56:33,611] INFO : plotted eta_target histogram +rank0 [2026-11-15 07:56:33,625] INFO : plotted eta_pred histogram +rank0 [2026-11-15 07:56:33,628] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 07:56:33,633] INFO : plotted sphi_target histogram +rank0 [2026-11-15 07:56:33,636] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 07:56:33,638] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 07:56:33,641] INFO : plotted cphi_target histogram +rank0 [2026-11-15 07:56:33,643] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 07:56:33,645] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 07:56:33,648] INFO : plotted energy_target histogram +rank0 [2026-11-15 07:56:33,651] INFO : plotted energy_pred histogram +rank0 [2026-11-15 07:56:33,654] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 07:56:41,326] INFO : VALIDATION | Step=73000/500000 | Train Loss=2.1618 | Valid Loss=2.5812 | Train Loss Unweighted=0.7444 | Valid Loss Unweighted=0.8445 | Stale=14 | ETA=1975.6m +rank0 [2026-11-15 07:56:41,328] INFO : split_configs=['1'] +rank0 [2026-11-15 07:56:41,338] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 07:56:41,346] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 07:56:41,458] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:56:42,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 07:56:42,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 07:56:43,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 07:56:43,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 07:56:43,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 07:56:44,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 07:56:44,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 07:56:44,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 07:56:45,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 07:56:45,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 07:56:45,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 07:56:46,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 07:56:46,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 07:56:46,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 07:56:47,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 07:56:47,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 07:56:47,708] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 07:56:47,712] INFO : split_configs=['1'] +rank0 [2026-11-15 07:56:47,723] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 07:56:47,726] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 07:56:47,808] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:56:49,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 07:56:49,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 07:56:49,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 07:56:50,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 07:56:50,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 07:56:51,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 07:56:51,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 07:56:52,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 07:56:52,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 07:56:53,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 07:56:53,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 07:56:53,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 07:56:54,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 07:56:54,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 07:56:55,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 07:56:55,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 07:56:55,861] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 07:56:55,864] INFO : split_configs=['1'] +rank0 [2026-11-15 07:56:55,875] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 07:56:55,879] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:56:55,963] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 07:56:57,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 07:56:57,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 07:56:57,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 07:56:58,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 07:56:58,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 07:56:59,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 07:56:59,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 07:56:59,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 07:57:00,391] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 07:57:00,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 07:57:01,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 07:57:01,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 07:57:02,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 07:57:02,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 07:57:02,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 07:57:03,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 07:57:03,393] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 07:57:03,401] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf +rank0 [2026-11-15 07:57:04,617] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_qq_pf +rank0 [2026-11-15 07:57:05,045] INFO : Plotted number of elements +rank0 [2026-11-15 07:57:06,358] INFO : Plotted jets +rank0 [2026-11-15 07:57:07,380] INFO : Plotted jet ratio +rank0 [2026-11-15 07:57:07,391] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:57:09,611] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ttbar_pf +rank0 [2026-11-15 07:57:10,064] INFO : Plotted number of elements +rank0 [2026-11-15 07:57:11,433] INFO : Plotted jets +rank0 [2026-11-15 07:57:12,488] INFO : Plotted jet ratio +rank0 [2026-11-15 07:57:12,497] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:57:14,003] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_73000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 07:57:14,448] INFO : Plotted number of elements +rank0 [2026-11-15 07:57:15,806] INFO : Plotted jets +rank0 [2026-11-15 07:57:16,796] INFO : Plotted jet ratio +rank0 [2026-11-15 07:59:13,999] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 07:59:13,999] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:01:12,102] INFO : Step 74000/500000 rank0 | Train Loss: 1.6976 | Train Loss Unweighted: 0.6749 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2263 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2121 | Regression_eta: 0.0185 | Regression_pt: 0.2026 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.2263 | Regression_cos_phi: 0.5385 | Regression_energy: 0.1270 | Regression_eta: 0.2407 | Regression_pt: 0.1212 | Regression_sin_phi: 0.3677 | LR: 2.55e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5046s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0625 | max_loss: 0.0029 | max_backward: 0.1338 | max_optimizer_step: 0.0963 | max_tensorboard_logging: 0.7545 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10645.0 | uss_MB: 9500.8 | child_rss_MB: 23865.6 | allocated_MB: 195.8 | reserved_MB: 20426.0 | inactive_split_MB: 158.2 +rank0 [2026-11-15 08:01:12,104] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-74000.pth +rank0 [2026-11-15 08:01:12,172] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 08:01:12,185] INFO : Running validation on rank0 +rank0 [2026-11-15 08:01:12,185] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:01:12,185] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:01:12,818] INFO : saved batch0_epoch74000.parquet +rank0 [2026-11-15 08:01:12,948] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:01:13,069] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:01:13,140] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:01:13,208] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:01:13,276] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:01:13,345] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:01:13,414] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:01:13,526] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:01:13,632] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:01:13,735] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:01:13,805] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:01:13,873] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:01:13,941] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:01:14,008] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:01:14,077] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:01:14,184] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:01:14,227] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:01:14,230] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:01:14,232] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:01:14,268] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:01:14,295] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:01:14,298] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:01:14,325] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:01:14,330] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:01:14,332] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:01:14,335] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:01:14,337] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:01:14,340] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:01:14,342] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:01:14,349] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:01:14,351] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:01:22,043] INFO : VALIDATION | Step=74000/500000 | Train Loss=1.6976 | Valid Loss=2.5508 | Train Loss Unweighted=0.6749 | Valid Loss Unweighted=0.8343 | Stale=15 | ETA=1971.3m +rank0 [2026-11-15 08:01:22,044] INFO : split_configs=['1'] +rank0 [2026-11-15 08:01:22,054] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:01:22,062] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:01:22,167] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:01:23,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:01:23,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:01:23,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:01:24,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:01:24,574] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:01:24,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:01:25,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:01:25,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:01:25,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:01:26,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:01:26,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:01:26,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:01:27,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:01:27,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:01:27,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:01:28,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:01:28,396] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:01:28,399] INFO : split_configs=['1'] +rank0 [2026-11-15 08:01:28,410] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:01:28,413] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:01:28,492] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:01:29,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:01:30,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:01:30,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:01:31,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:01:31,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:01:31,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:01:32,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:01:32,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:01:33,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:01:33,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:01:34,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:01:34,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:01:35,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:01:35,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:01:35,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:01:36,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:01:36,438] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:01:36,441] INFO : split_configs=['1'] +rank0 [2026-11-15 08:01:36,452] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:01:36,455] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:01:36,535] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:01:37,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:01:38,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:01:38,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:01:38,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:01:39,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:01:39,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:01:40,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:01:40,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:01:40,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:01:41,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:01:41,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:01:42,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:01:42,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:01:42,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:01:43,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:01:43,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:01:43,900] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:01:43,909] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf +rank0 [2026-11-15 08:01:45,113] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_qq_pf +rank0 [2026-11-15 08:01:45,543] INFO : Plotted number of elements +rank0 [2026-11-15 08:01:46,827] INFO : Plotted jets +rank0 [2026-11-15 08:01:47,859] INFO : Plotted jet ratio +rank0 [2026-11-15 08:01:47,869] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:01:49,557] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:01:49,995] INFO : Plotted number of elements +rank0 [2026-11-15 08:01:51,904] INFO : Plotted jets +rank0 [2026-11-15 08:01:52,942] INFO : Plotted jet ratio +rank0 [2026-11-15 08:01:52,951] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:01:54,398] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_74000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:01:54,836] INFO : Plotted number of elements +rank0 [2026-11-15 08:01:56,165] INFO : Plotted jets +rank0 [2026-11-15 08:01:57,147] INFO : Plotted jet ratio +rank0 [2026-11-15 08:05:50,676] INFO : Step 75000/500000 rank0 | Train Loss: 1.5745 | Train Loss Unweighted: 0.7623 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2283 | Regression_cos_phi: 0.0084 | Regression_energy: 0.2569 | Regression_eta: 0.0147 | Regression_pt: 0.2491 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2283 | Regression_cos_phi: 0.4912 | Regression_energy: 0.1537 | Regression_eta: 0.1912 | Regression_pt: 0.1490 | Regression_sin_phi: 0.2912 | LR: 2.60e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4887s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0027 | max_forward: 0.0637 | max_loss: 0.0034 | max_backward: 0.1351 | max_optimizer_step: 0.0954 | max_tensorboard_logging: 0.7848 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 10607.3 | uss_MB: 9463.2 | child_rss_MB: 23811.9 | allocated_MB: 195.9 | reserved_MB: 17986.0 | inactive_split_MB: 240.1 +rank0 [2026-11-15 08:05:50,678] INFO : Running validation on rank0 +rank0 [2026-11-15 08:05:50,678] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:05:50,678] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:05:51,313] INFO : saved batch0_epoch75000.parquet +rank0 [2026-11-15 08:05:51,437] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:05:51,552] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:05:51,623] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:05:51,691] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:05:51,759] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:05:51,828] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:05:51,897] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:05:52,009] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:05:52,119] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:05:52,225] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:05:52,295] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:05:52,363] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:05:52,429] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:05:52,497] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:05:52,564] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:05:52,669] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:05:52,676] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:05:52,679] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:05:52,687] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:05:52,690] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:05:52,693] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:05:52,695] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:05:52,697] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:05:52,700] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:05:52,702] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:05:52,705] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:05:52,707] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:05:52,710] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:05:52,712] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:05:52,715] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:05:52,717] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:06:00,442] INFO : VALIDATION | Step=75000/500000 | Train Loss=1.5745 | Valid Loss=2.5886 | Train Loss Unweighted=0.7623 | Valid Loss Unweighted=0.8278 | Stale=16 | ETA=1966.7m +rank0 [2026-11-15 08:06:00,442] INFO : split_configs=['1'] +rank0 [2026-11-15 08:06:00,454] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:06:00,460] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:06:00,578] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:06:01,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:06:02,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:06:02,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:06:02,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:06:03,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:06:03,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:06:03,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:06:04,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:06:04,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:06:04,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:06:05,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:06:05,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:06:05,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:06:06,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:06:06,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:06:06,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:06:06,940] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:06:06,943] INFO : split_configs=['1'] +rank0 [2026-11-15 08:06:06,954] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:06:06,958] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:06:07,042] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:06:08,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:06:08,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:06:09,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:06:09,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:06:10,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:06:10,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:06:10,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:06:11,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:06:11,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:06:12,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:06:12,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:06:13,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:06:13,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:06:14,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:06:14,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:06:15,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:06:15,145] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:06:15,148] INFO : split_configs=['1'] +rank0 [2026-11-15 08:06:15,158] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:06:15,162] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:06:15,248] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:06:16,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:06:16,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:06:17,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:06:17,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:06:18,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:06:18,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:06:18,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:06:19,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:06:19,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:06:20,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:06:20,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:06:20,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:06:21,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:06:21,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:06:22,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:06:22,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:06:22,678] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:06:22,686] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf +rank0 [2026-11-15 08:06:23,896] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_qq_pf +rank0 [2026-11-15 08:06:24,333] INFO : Plotted number of elements +rank0 [2026-11-15 08:06:25,606] INFO : Plotted jets +rank0 [2026-11-15 08:06:26,645] INFO : Plotted jet ratio +rank0 [2026-11-15 08:06:26,655] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:06:28,280] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:06:29,295] INFO : Plotted number of elements +rank0 [2026-11-15 08:06:30,627] INFO : Plotted jets +rank0 [2026-11-15 08:06:31,670] INFO : Plotted jet ratio +rank0 [2026-11-15 08:06:31,679] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:06:33,160] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_75000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:06:33,597] INFO : Plotted number of elements +rank0 [2026-11-15 08:06:34,926] INFO : Plotted jets +rank0 [2026-11-15 08:06:35,910] INFO : Plotted jet ratio +rank0 [2026-11-15 08:06:36,406] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 08:06:36,406] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:10:31,141] INFO : Step 76000/500000 rank0 | Train Loss: 2.1588 | Train Loss Unweighted: 0.7861 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2232 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2658 | Regression_eta: 0.0345 | Regression_pt: 0.2439 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2232 | Regression_cos_phi: 0.5431 | Regression_energy: 0.1591 | Regression_eta: 0.4482 | Regression_pt: 0.1459 | Regression_sin_phi: 0.5787 | LR: 2.65e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.5725s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0546 | max_loss: 0.0029 | max_backward: 0.1149 | max_optimizer_step: 0.0979 | max_tensorboard_logging: 1.8890 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10673.2 | uss_MB: 9529.0 | child_rss_MB: 23733.4 | allocated_MB: 196.4 | reserved_MB: 18368.0 | inactive_split_MB: 257.6 +rank0 [2026-11-15 08:10:31,144] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-76000.pth +rank0 [2026-11-15 08:10:31,214] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 08:10:31,229] INFO : Running validation on rank0 +rank0 [2026-11-15 08:10:31,229] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:10:31,230] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:10:31,855] INFO : saved batch0_epoch76000.parquet +rank0 [2026-11-15 08:10:31,975] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:10:32,092] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:10:32,161] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:10:32,229] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:10:32,295] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:10:32,362] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:10:32,429] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:10:32,537] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:10:32,641] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:10:32,744] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:10:32,813] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:10:32,881] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:10:32,949] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:10:33,019] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:10:33,087] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:10:33,193] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:10:33,213] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:10:33,225] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:10:33,228] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:10:33,231] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:10:33,238] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:10:33,240] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:10:33,243] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:10:33,245] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:10:33,248] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:10:33,250] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:10:33,253] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:10:33,255] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:10:33,258] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:10:33,260] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:10:33,263] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:10:40,946] INFO : VALIDATION | Step=76000/500000 | Train Loss=2.1588 | Valid Loss=2.6047 | Train Loss Unweighted=0.7861 | Valid Loss Unweighted=0.8316 | Stale=17 | ETA=1962.4m +rank0 [2026-11-15 08:10:40,947] INFO : split_configs=['1'] +rank0 [2026-11-15 08:10:40,963] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:10:40,972] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:10:41,089] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:10:42,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:10:42,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:10:42,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:10:43,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:10:43,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:10:43,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:10:44,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:10:44,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:10:44,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:10:45,251] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:10:45,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:10:45,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:10:46,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:10:46,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:10:46,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:10:47,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:10:47,361] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:10:47,364] INFO : split_configs=['1'] +rank0 [2026-11-15 08:10:47,382] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:10:47,385] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:10:47,469] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:10:48,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:10:49,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:10:49,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:10:50,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:10:50,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:10:50,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:10:51,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:10:51,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:10:52,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:10:52,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:10:53,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:10:53,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:10:54,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:10:54,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:10:54,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:10:55,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:10:55,550] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:10:55,553] INFO : split_configs=['1'] +rank0 [2026-11-15 08:10:55,571] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:10:55,576] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:10:55,662] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:10:56,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:10:57,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:10:57,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:10:58,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:10:58,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:10:58,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:10:59,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:10:59,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:11:00,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:11:00,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:11:00,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:11:01,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:11:01,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:11:02,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:11:02,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:11:03,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:11:03,166] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:11:03,174] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf +rank0 [2026-11-15 08:11:04,364] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_qq_pf +rank0 [2026-11-15 08:11:04,793] INFO : Plotted number of elements +rank0 [2026-11-15 08:11:06,110] INFO : Plotted jets +rank0 [2026-11-15 08:11:07,137] INFO : Plotted jet ratio +rank0 [2026-11-15 08:11:07,162] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:11:08,978] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:11:10,002] INFO : Plotted number of elements +rank0 [2026-11-15 08:11:11,351] INFO : Plotted jets +rank0 [2026-11-15 08:11:12,419] INFO : Plotted jet ratio +rank0 [2026-11-15 08:11:12,428] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:11:13,918] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_76000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:11:14,367] INFO : Plotted number of elements +rank0 [2026-11-15 08:11:15,726] INFO : Plotted jets +rank0 [2026-11-15 08:11:16,722] INFO : Plotted jet ratio +rank0 [2026-11-15 08:13:13,338] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 08:13:13,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:15:11,607] INFO : Step 77000/500000 rank0 | Train Loss: 1.7834 | Train Loss Unweighted: 0.7779 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2328 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2549 | Regression_eta: 0.0127 | Regression_pt: 0.2611 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.1095 | Classification_binary: 0.2328 | Regression_cos_phi: 0.5244 | Regression_energy: 0.1525 | Regression_eta: 0.1651 | Regression_pt: 0.1562 | Regression_sin_phi: 0.4428 | LR: 2.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.4213s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0582 | max_loss: 0.0029 | max_backward: 0.1203 | max_optimizer_step: 0.0949 | max_tensorboard_logging: 1.7348 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 9322.9 | uss_MB: 8178.8 | child_rss_MB: 23821.5 | allocated_MB: 195.2 | reserved_MB: 18500.0 | inactive_split_MB: 292.8 +rank0 [2026-11-15 08:15:11,612] INFO : Running validation on rank0 +rank0 [2026-11-15 08:15:11,612] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:15:11,612] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:15:12,245] INFO : saved batch0_epoch77000.parquet +rank0 [2026-11-15 08:15:12,375] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:15:12,494] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:15:12,566] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:15:12,639] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:15:12,709] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:15:12,777] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:15:12,846] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:15:12,957] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:15:13,065] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:15:13,170] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:15:13,240] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:15:13,308] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:15:13,375] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:15:13,443] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:15:13,510] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:15:13,614] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:15:13,621] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:15:13,624] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:15:13,627] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:15:13,629] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:15:13,632] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:15:13,634] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:15:13,638] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:15:13,640] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:15:13,643] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:15:13,645] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:15:13,648] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:15:13,650] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:15:13,652] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:15:13,655] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:15:13,657] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:15:21,399] INFO : VALIDATION | Step=77000/500000 | Train Loss=1.7834 | Valid Loss=2.5878 | Train Loss Unweighted=0.7779 | Valid Loss Unweighted=0.8254 | Stale=18 | ETA=1958.0m +rank0 [2026-11-15 08:15:21,399] INFO : split_configs=['1'] +rank0 [2026-11-15 08:15:21,417] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:15:21,426] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:15:21,531] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:15:22,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:15:22,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:15:23,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:15:23,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:15:23,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:15:24,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:15:24,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:15:24,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:15:25,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:15:25,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:15:26,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:15:26,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:15:26,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:15:27,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:15:27,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:15:27,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:15:27,852] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:15:27,855] INFO : split_configs=['1'] +rank0 [2026-11-15 08:15:27,875] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:15:27,878] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:15:27,956] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:15:29,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:15:29,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:15:30,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:15:30,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:15:30,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:15:31,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:15:31,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:15:32,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:15:32,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:15:33,171] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:15:33,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:15:34,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:15:34,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:15:34,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:15:35,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:15:35,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:15:35,954] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:15:35,958] INFO : split_configs=['1'] +rank0 [2026-11-15 08:15:35,975] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:15:35,979] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:15:36,062] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:15:37,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:15:37,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:15:38,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:15:38,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:15:38,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:15:39,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:15:39,701] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:15:40,107] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:15:40,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:15:40,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:15:41,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:15:41,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:15:42,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:15:42,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:15:43,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:15:43,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:15:43,533] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:15:43,553] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf +rank0 [2026-11-15 08:15:44,784] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_qq_pf +rank0 [2026-11-15 08:15:45,220] INFO : Plotted number of elements +rank0 [2026-11-15 08:15:46,536] INFO : Plotted jets +rank0 [2026-11-15 08:15:47,560] INFO : Plotted jet ratio +rank0 [2026-11-15 08:15:47,585] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:15:49,330] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:15:50,342] INFO : Plotted number of elements +rank0 [2026-11-15 08:15:51,708] INFO : Plotted jets +rank0 [2026-11-15 08:15:52,753] INFO : Plotted jet ratio +rank0 [2026-11-15 08:15:52,763] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:15:54,281] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_77000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:15:54,723] INFO : Plotted number of elements +rank0 [2026-11-15 08:15:56,064] INFO : Plotted jets +rank0 [2026-11-15 08:15:57,057] INFO : Plotted jet ratio +rank0 [2026-11-15 08:19:52,839] INFO : Step 78000/500000 rank0 | Train Loss: 1.6543 | Train Loss Unweighted: 0.8745 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2238 | Regression_cos_phi: 0.0057 | Regression_energy: 0.3213 | Regression_eta: 0.0268 | Regression_pt: 0.2919 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0877 | Classification_binary: 0.2238 | Regression_cos_phi: 0.3377 | Regression_energy: 0.1923 | Regression_eta: 0.3476 | Regression_pt: 0.1746 | Regression_sin_phi: 0.2906 | LR: 2.75e-04 | DataLoad Time: 0.0021s | Model Forward Time: 2.8878s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0027 | max_forward: 0.0585 | max_loss: 0.0029 | max_backward: 0.1211 | max_optimizer_step: 0.0955 | max_tensorboard_logging: 2.1938 | max_tensorboard_flush: 0.0008 | Memory: rss_MB: 10814.0 | uss_MB: 9669.8 | child_rss_MB: 23777.0 | allocated_MB: 196.9 | reserved_MB: 20432.0 | inactive_split_MB: 287.1 +rank0 [2026-11-15 08:19:52,846] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-78000.pth +rank0 [2026-11-15 08:19:52,927] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 08:19:52,942] INFO : Running validation on rank0 +rank0 [2026-11-15 08:19:52,942] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:19:52,942] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:19:53,572] INFO : saved batch0_epoch78000.parquet +rank0 [2026-11-15 08:19:53,693] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:19:53,807] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:19:53,877] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:19:53,945] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:19:54,012] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:19:54,081] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:19:54,148] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:19:54,263] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:19:54,367] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:19:54,469] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:19:54,538] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:19:54,607] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:19:54,674] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:19:54,743] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:19:54,812] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:19:54,919] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:19:54,938] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:19:54,945] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:19:54,947] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:19:54,951] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:19:54,954] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:19:54,957] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:19:54,959] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:19:54,962] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:19:54,964] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:19:54,966] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:19:54,969] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:19:54,972] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:19:54,974] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:19:54,977] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:19:54,979] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:20:02,691] INFO : VALIDATION | Step=78000/500000 | Train Loss=1.6543 | Valid Loss=2.6549 | Train Loss Unweighted=0.8745 | Valid Loss Unweighted=0.8375 | Stale=19 | ETA=1953.7m +rank0 [2026-11-15 08:20:02,691] INFO : split_configs=['1'] +rank0 [2026-11-15 08:20:02,708] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:20:02,716] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:20:03,429] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:20:04,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:20:04,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:20:05,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:20:05,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:20:05,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:20:06,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:20:06,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:20:06,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:20:07,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:20:07,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:20:08,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:20:08,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:20:08,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:20:09,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:20:09,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:20:09,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:20:09,800] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:20:09,803] INFO : split_configs=['1'] +rank0 [2026-11-15 08:20:09,821] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:20:09,835] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:20:09,921] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:20:11,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:20:11,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:20:12,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:20:12,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:20:13,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:20:13,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:20:13,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:20:14,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:20:14,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:20:15,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:20:15,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:20:16,236] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:20:16,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:20:17,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:20:17,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:20:18,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:20:18,149] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:20:18,153] INFO : split_configs=['1'] +rank0 [2026-11-15 08:20:18,171] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:20:18,185] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:20:18,273] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:20:19,554] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:20:19,979] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:20:20,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:20:20,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:20:21,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:20:21,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:20:22,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:20:22,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:20:22,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:20:23,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:20:23,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:20:24,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:20:24,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:20:24,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:20:25,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:20:25,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:20:25,838] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:20:26,170] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf +rank0 [2026-11-15 08:20:27,384] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_qq_pf +rank0 [2026-11-15 08:20:27,839] INFO : Plotted number of elements +rank0 [2026-11-15 08:20:29,111] INFO : Plotted jets +rank0 [2026-11-15 08:20:30,159] INFO : Plotted jet ratio +rank0 [2026-11-15 08:20:30,170] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:20:31,843] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:20:32,289] INFO : Plotted number of elements +rank0 [2026-11-15 08:20:34,201] INFO : Plotted jets +rank0 [2026-11-15 08:20:35,246] INFO : Plotted jet ratio +rank0 [2026-11-15 08:20:35,257] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:20:36,718] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_78000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:20:37,156] INFO : Plotted number of elements +rank0 [2026-11-15 08:20:38,488] INFO : Plotted jets +rank0 [2026-11-15 08:20:39,474] INFO : Plotted jet ratio +rank0 [2026-11-15 08:20:39,955] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 08:20:39,955] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:24:34,778] INFO : Step 79000/500000 rank0 | Train Loss: 1.7156 | Train Loss Unweighted: 0.9099 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.2634 | Regression_cos_phi: 0.0072 | Regression_energy: 0.3109 | Regression_eta: 0.0124 | Regression_pt: 0.3093 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0955 | Classification_binary: 0.2634 | Regression_cos_phi: 0.4241 | Regression_energy: 0.1861 | Regression_eta: 0.1608 | Regression_pt: 0.1850 | Regression_sin_phi: 0.4008 | LR: 2.80e-04 | DataLoad Time: 0.0024s | Model Forward Time: 2.7678s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0025 | max_forward: 0.0590 | max_loss: 0.0028 | max_backward: 0.1241 | max_optimizer_step: 0.0950 | max_tensorboard_logging: 2.0671 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 10902.3 | uss_MB: 9758.2 | child_rss_MB: 23650.8 | allocated_MB: 195.8 | reserved_MB: 18684.0 | inactive_split_MB: 152.2 +rank0 [2026-11-15 08:24:34,780] INFO : Running validation on rank0 +rank0 [2026-11-15 08:24:34,781] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:24:34,781] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:24:35,419] INFO : saved batch0_epoch79000.parquet +rank0 [2026-11-15 08:24:35,541] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:24:35,658] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:24:35,729] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:24:35,797] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:24:35,864] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:24:35,932] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:24:36,000] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:24:36,114] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:24:36,219] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:24:36,321] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:24:36,389] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:24:36,457] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:24:36,522] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:24:36,588] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:24:36,654] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:24:36,759] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:24:36,802] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:24:36,829] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:24:36,832] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:24:36,859] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:24:36,886] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:24:36,889] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:24:36,911] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:24:36,914] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:24:36,916] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:24:36,918] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:24:36,921] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:24:36,926] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:24:36,928] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:24:36,937] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:24:36,941] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:24:44,654] INFO : VALIDATION | Step=79000/500000 | Train Loss=1.7156 | Valid Loss=2.6497 | Train Loss Unweighted=0.9099 | Valid Loss Unweighted=0.8442 | Stale=20 | ETA=1949.4m +rank0 [2026-11-15 08:24:44,656] INFO : split_configs=['1'] +rank0 [2026-11-15 08:24:44,673] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:24:44,682] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:24:44,798] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:24:45,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:24:46,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:24:46,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:24:46,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:24:47,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:24:47,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:24:47,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:24:48,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:24:48,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:24:49,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:24:49,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:24:49,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:24:50,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:24:50,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:24:50,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:24:51,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:24:51,136] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:24:51,140] INFO : split_configs=['1'] +rank0 [2026-11-15 08:24:51,158] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:24:51,165] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:24:51,251] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:24:52,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:24:52,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:24:53,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:24:53,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:24:54,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:24:54,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:24:55,142] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:24:55,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:24:56,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:24:56,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:24:56,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:24:57,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:24:57,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:24:58,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:24:58,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:24:59,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:24:59,309] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:24:59,313] INFO : split_configs=['1'] +rank0 [2026-11-15 08:24:59,329] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:24:59,345] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:24:59,432] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:25:00,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:25:01,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:25:01,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:25:01,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:25:02,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:25:02,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:25:03,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:25:03,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:25:03,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:25:04,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:25:04,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:25:05,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:25:05,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:25:06,073] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:25:06,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:25:06,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:25:07,011] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:25:07,030] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf +rank0 [2026-11-15 08:25:08,257] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_qq_pf +rank0 [2026-11-15 08:25:08,684] INFO : Plotted number of elements +rank0 [2026-11-15 08:25:10,012] INFO : Plotted jets +rank0 [2026-11-15 08:25:11,034] INFO : Plotted jet ratio +rank0 [2026-11-15 08:25:11,046] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:25:13,306] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:25:13,757] INFO : Plotted number of elements +rank0 [2026-11-15 08:25:15,125] INFO : Plotted jets +rank0 [2026-11-15 08:25:16,163] INFO : Plotted jet ratio +rank0 [2026-11-15 08:25:16,200] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:25:17,809] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_79000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:25:18,257] INFO : Plotted number of elements +rank0 [2026-11-15 08:25:19,606] INFO : Plotted jets +rank0 [2026-11-15 08:25:20,598] INFO : Plotted jet ratio +rank0 [2026-11-15 08:27:16,969] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 08:27:16,969] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:29:15,201] INFO : Step 80000/500000 rank0 | Train Loss: 1.5259 | Train Loss Unweighted: 0.8379 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2344 | Regression_cos_phi: 0.0047 | Regression_energy: 0.2806 | Regression_eta: 0.0311 | Regression_pt: 0.2838 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0848 | Classification_binary: 0.2344 | Regression_cos_phi: 0.2748 | Regression_energy: 0.1680 | Regression_eta: 0.4036 | Regression_pt: 0.1697 | Regression_sin_phi: 0.1906 | LR: 2.85e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.3365s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0027 | max_forward: 0.0609 | max_loss: 0.0029 | max_backward: 0.1302 | max_optimizer_step: 0.0945 | max_tensorboard_logging: 1.6353 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 10668.0 | uss_MB: 9530.4 | child_rss_MB: 23834.4 | allocated_MB: 195.5 | reserved_MB: 20714.0 | inactive_split_MB: 236.5 +rank0 [2026-11-15 08:29:15,204] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-80000.pth +rank0 [2026-11-15 08:29:15,275] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 08:29:15,293] INFO : Running validation on rank0 +rank0 [2026-11-15 08:29:15,293] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:29:15,293] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:29:15,941] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-15 08:29:16,064] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:29:16,180] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:29:16,251] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:29:16,319] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:29:16,386] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:29:16,454] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:29:16,522] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:29:16,633] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:29:16,738] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:29:16,841] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:29:16,910] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:29:16,979] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:29:17,047] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:29:17,116] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:29:17,186] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:29:17,294] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:29:17,315] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:29:17,327] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:29:17,330] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:29:17,340] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:29:17,353] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:29:17,356] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:29:17,366] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:29:17,379] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:29:17,382] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:29:17,384] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:29:17,387] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:29:17,389] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:29:17,391] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:29:17,394] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:29:17,397] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:29:25,074] INFO : VALIDATION | Step=80000/500000 | Train Loss=1.5259 | Valid Loss=2.5604 | Train Loss Unweighted=0.8379 | Valid Loss Unweighted=0.8351 | Stale=21 | ETA=1945.0m +rank0 [2026-11-15 08:29:25,074] INFO : split_configs=['1'] +rank0 [2026-11-15 08:29:25,088] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:29:25,097] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:29:25,211] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:29:26,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:29:26,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:29:26,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:29:27,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:29:27,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:29:28,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:29:28,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:29:28,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:29:29,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:29:29,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:29:29,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:29:30,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:29:30,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:29:30,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:29:31,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:29:31,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:29:31,547] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:29:31,551] INFO : split_configs=['1'] +rank0 [2026-11-15 08:29:31,568] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:29:31,581] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:29:31,667] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:29:32,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:29:33,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:29:33,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:29:34,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:29:34,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:29:35,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:29:35,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:29:35,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:29:36,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:29:36,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:29:37,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:29:37,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:29:38,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:29:38,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:29:39,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:29:39,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:29:39,707] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:29:39,710] INFO : split_configs=['1'] +rank0 [2026-11-15 08:29:39,727] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:29:39,731] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:29:39,819] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:29:41,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:29:41,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:29:41,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:29:42,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:29:42,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:29:43,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:29:43,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:29:43,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:29:44,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:29:44,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:29:45,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:29:45,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:29:45,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:29:46,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:29:46,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:29:47,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:29:47,245] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:29:47,253] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf +rank0 [2026-11-15 08:29:48,501] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_qq_pf +rank0 [2026-11-15 08:29:48,918] INFO : Plotted number of elements +rank0 [2026-11-15 08:29:50,209] INFO : Plotted jets +rank0 [2026-11-15 08:29:51,224] INFO : Plotted jet ratio +rank0 [2026-11-15 08:29:51,235] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:29:52,886] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:29:53,323] INFO : Plotted number of elements +rank0 [2026-11-15 08:29:55,231] INFO : Plotted jets +rank0 [2026-11-15 08:29:56,275] INFO : Plotted jet ratio +rank0 [2026-11-15 08:29:56,284] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:29:57,724] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_80000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:29:58,161] INFO : Plotted number of elements +rank0 [2026-11-15 08:29:59,533] INFO : Plotted jets +rank0 [2026-11-15 08:30:00,526] INFO : Plotted jet ratio +rank0 [2026-11-15 08:33:55,712] INFO : Step 81000/500000 rank0 | Train Loss: 1.7250 | Train Loss Unweighted: 0.8072 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2352 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2679 | Regression_eta: 0.0108 | Regression_pt: 0.2775 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0800 | Classification_binary: 0.2352 | Regression_cos_phi: 0.5009 | Regression_energy: 0.1603 | Regression_eta: 0.1401 | Regression_pt: 0.1660 | Regression_sin_phi: 0.4425 | LR: 2.90e-04 | DataLoad Time: 0.0023s | Model Forward Time: 3.0514s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0712 | max_loss: 0.0040 | max_backward: 0.1531 | max_optimizer_step: 0.0967 | max_tensorboard_logging: 2.3037 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11126.5 | uss_MB: 9988.9 | child_rss_MB: 23790.1 | allocated_MB: 200.2 | reserved_MB: 21586.0 | inactive_split_MB: 461.8 +rank0 [2026-11-15 08:33:55,714] INFO : Running validation on rank0 +rank0 [2026-11-15 08:33:55,714] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:33:55,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:33:56,361] INFO : saved batch0_epoch81000.parquet +rank0 [2026-11-15 08:33:56,484] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:33:56,601] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:33:56,673] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:33:56,741] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:33:56,810] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:33:56,880] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:33:56,951] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:33:57,061] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:33:57,165] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:33:57,269] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:33:57,338] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:33:57,405] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:33:57,471] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:33:57,537] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:33:57,604] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:33:57,707] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:33:57,714] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:33:57,717] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:33:57,719] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:33:57,722] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:33:57,725] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:33:57,727] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:33:57,730] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:33:57,732] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:33:57,735] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:33:57,737] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:33:57,740] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:33:57,742] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:33:57,744] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:33:57,747] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:33:57,749] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:34:05,462] INFO : VALIDATION | Step=81000/500000 | Train Loss=1.7250 | Valid Loss=2.5705 | Train Loss Unweighted=0.8072 | Valid Loss Unweighted=0.8231 | Stale=22 | ETA=1940.6m +rank0 [2026-11-15 08:34:05,463] INFO : split_configs=['1'] +rank0 [2026-11-15 08:34:05,481] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:34:05,491] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:34:05,610] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:34:06,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:34:07,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:34:07,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:34:07,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:34:08,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:34:08,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:34:08,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:34:09,115] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:34:09,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:34:09,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:34:10,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:34:10,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:34:10,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:34:11,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:34:11,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:34:11,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:34:11,939] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:34:11,942] INFO : split_configs=['1'] +rank0 [2026-11-15 08:34:11,960] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:34:11,964] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:34:12,051] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:34:13,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:34:13,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:34:14,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:34:14,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:34:15,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:34:15,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:34:15,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:34:16,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:34:16,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:34:17,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:34:17,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:34:18,276] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:34:18,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:34:19,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:34:19,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:34:20,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:34:20,240] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:34:20,244] INFO : split_configs=['1'] +rank0 [2026-11-15 08:34:20,262] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:34:20,265] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:34:20,354] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:34:21,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:34:22,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:34:22,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:34:22,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:34:23,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:34:23,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:34:24,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:34:24,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:34:24,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:34:25,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:34:25,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:34:26,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:34:26,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:34:26,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:34:27,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:34:27,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:34:27,870] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:34:27,880] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf +rank0 [2026-11-15 08:34:29,112] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_qq_pf +rank0 [2026-11-15 08:34:29,531] INFO : Plotted number of elements +rank0 [2026-11-15 08:34:30,836] INFO : Plotted jets +rank0 [2026-11-15 08:34:31,854] INFO : Plotted jet ratio +rank0 [2026-11-15 08:34:31,878] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:34:33,515] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:34:33,956] INFO : Plotted number of elements +rank0 [2026-11-15 08:34:35,878] INFO : Plotted jets +rank0 [2026-11-15 08:34:36,916] INFO : Plotted jet ratio +rank0 [2026-11-15 08:34:36,937] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:34:38,398] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_81000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:34:38,859] INFO : Plotted number of elements +rank0 [2026-11-15 08:34:40,188] INFO : Plotted jets +rank0 [2026-11-15 08:34:41,186] INFO : Plotted jet ratio +rank0 [2026-11-15 08:34:41,675] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 08:34:41,675] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:38:35,769] INFO : Step 82000/500000 rank0 | Train Loss: 2.7318 | Train Loss Unweighted: 0.8209 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2615 | Regression_cos_phi: 0.0109 | Regression_energy: 0.2478 | Regression_eta: 0.0494 | Regression_pt: 0.2382 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0937 | Classification_binary: 0.2615 | Regression_cos_phi: 0.6382 | Regression_energy: 0.1483 | Regression_eta: 0.6421 | Regression_pt: 0.1425 | Regression_sin_phi: 0.8055 | LR: 2.95e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.7294s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0650 | max_loss: 0.0029 | max_backward: 0.1376 | max_optimizer_step: 0.0970 | max_tensorboard_logging: 1.9982 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11233.9 | uss_MB: 10096.3 | child_rss_MB: 23748.8 | allocated_MB: 198.9 | reserved_MB: 19538.0 | inactive_split_MB: 405.1 +rank0 [2026-11-15 08:38:35,771] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-82000.pth +rank0 [2026-11-15 08:38:35,842] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 08:38:35,857] INFO : Running validation on rank0 +rank0 [2026-11-15 08:38:35,857] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:38:35,857] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:38:36,481] INFO : saved batch0_epoch82000.parquet +rank0 [2026-11-15 08:38:36,603] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:38:36,724] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:38:36,795] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:38:36,865] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:38:36,933] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:38:37,003] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:38:37,073] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:38:37,187] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:38:37,291] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:38:37,393] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:38:37,463] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:38:37,530] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:38:37,595] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:38:37,661] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:38:37,727] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:38:37,830] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:38:37,837] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:38:37,840] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:38:37,843] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:38:37,845] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:38:37,848] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:38:37,851] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:38:37,853] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:38:37,856] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:38:37,858] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:38:37,860] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:38:37,863] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:38:37,865] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:38:37,907] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:38:37,911] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:38:37,913] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:38:45,586] INFO : VALIDATION | Step=82000/500000 | Train Loss=2.7318 | Valid Loss=2.5645 | Train Loss Unweighted=0.8209 | Valid Loss Unweighted=0.8237 | Stale=23 | ETA=1936.2m +rank0 [2026-11-15 08:38:45,587] INFO : split_configs=['1'] +rank0 [2026-11-15 08:38:45,604] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:38:45,616] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:38:45,723] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:38:46,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:38:47,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:38:47,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:38:47,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:38:48,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:38:48,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:38:48,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:38:49,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:38:49,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:38:49,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:38:50,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:38:50,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:38:50,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:38:51,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:38:51,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:38:51,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:38:51,940] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:38:51,943] INFO : split_configs=['1'] +rank0 [2026-11-15 08:38:51,962] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:38:51,965] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:38:52,051] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:38:53,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:38:53,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:38:54,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:38:54,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:38:54,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:38:55,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:38:55,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:38:56,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:38:56,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:38:57,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:38:57,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:38:58,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:38:58,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:38:59,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:38:59,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:38:59,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:39:00,003] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:39:00,007] INFO : split_configs=['1'] +rank0 [2026-11-15 08:39:00,024] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:39:00,027] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:39:00,117] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:39:01,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:39:01,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:39:02,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:39:02,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:39:02,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:39:03,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:39:03,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:39:04,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:39:04,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:39:04,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:39:05,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:39:05,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:39:06,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:39:06,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:39:07,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:39:07,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:39:07,550] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:39:07,558] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf +rank0 [2026-11-15 08:39:08,812] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_qq_pf +rank0 [2026-11-15 08:39:09,251] INFO : Plotted number of elements +rank0 [2026-11-15 08:39:10,551] INFO : Plotted jets +rank0 [2026-11-15 08:39:11,573] INFO : Plotted jet ratio +rank0 [2026-11-15 08:39:11,586] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:39:13,301] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:39:13,758] INFO : Plotted number of elements +rank0 [2026-11-15 08:39:15,692] INFO : Plotted jets +rank0 [2026-11-15 08:39:16,732] INFO : Plotted jet ratio +rank0 [2026-11-15 08:39:16,743] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:39:18,224] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_82000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:39:18,665] INFO : Plotted number of elements +rank0 [2026-11-15 08:39:20,002] INFO : Plotted jets +rank0 [2026-11-15 08:39:20,987] INFO : Plotted jet ratio +rank0 [2026-11-15 08:41:17,503] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 08:41:17,504] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:43:16,419] INFO : Step 83000/500000 rank0 | Train Loss: 1.3647 | Train Loss Unweighted: 0.6794 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2196 | Regression_cos_phi: 0.0035 | Regression_energy: 0.2203 | Regression_eta: 0.0255 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2196 | Regression_cos_phi: 0.2083 | Regression_energy: 0.1319 | Regression_eta: 0.3306 | Regression_pt: 0.1231 | Regression_sin_phi: 0.2761 | LR: 3.00e-04 | DataLoad Time: 0.0025s | Model Forward Time: 2.5498s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0028 | max_forward: 0.0724 | max_loss: 0.0028 | max_backward: 0.1593 | max_optimizer_step: 0.0975 | max_tensorboard_logging: 1.7920 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 11269.2 | uss_MB: 10131.6 | child_rss_MB: 23846.5 | allocated_MB: 196.9 | reserved_MB: 19362.0 | inactive_split_MB: 303.1 +rank0 [2026-11-15 08:43:16,423] INFO : Running validation on rank0 +rank0 [2026-11-15 08:43:16,424] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:43:16,424] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:43:17,047] INFO : saved batch0_epoch83000.parquet +rank0 [2026-11-15 08:43:17,171] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:43:17,284] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:43:17,354] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:43:17,422] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:43:17,490] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:43:17,559] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:43:17,628] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:43:17,739] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:43:17,847] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:43:17,952] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:43:18,022] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:43:18,089] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:43:18,155] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:43:18,221] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:43:18,287] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:43:18,390] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:43:18,398] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:43:18,400] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:43:18,403] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:43:18,405] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:43:18,408] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:43:18,411] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:43:18,413] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:43:18,416] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:43:18,418] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:43:18,420] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:43:18,423] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:43:18,425] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:43:18,428] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:43:18,430] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:43:18,433] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:43:26,134] INFO : VALIDATION | Step=83000/500000 | Train Loss=1.3647 | Valid Loss=2.6542 | Train Loss Unweighted=0.6794 | Valid Loss Unweighted=0.8275 | Stale=24 | ETA=1931.7m +rank0 [2026-11-15 08:43:26,134] INFO : split_configs=['1'] +rank0 [2026-11-15 08:43:26,148] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:43:26,155] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:43:26,273] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:43:27,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:43:27,701] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:43:28,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:43:28,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:43:28,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:43:29,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:43:29,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:43:29,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:43:30,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:43:30,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:43:30,796] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:43:31,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:43:31,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:43:31,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:43:32,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:43:32,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:43:32,562] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:43:32,565] INFO : split_configs=['1'] +rank0 [2026-11-15 08:43:32,583] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:43:32,586] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:43:32,676] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:43:33,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:43:34,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:43:34,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:43:35,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:43:35,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:43:36,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:43:36,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:43:37,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:43:37,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:43:38,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:43:38,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:43:38,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:43:39,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:43:39,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:43:40,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:43:40,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:43:40,794] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:43:40,797] INFO : split_configs=['1'] +rank0 [2026-11-15 08:43:40,813] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:43:40,817] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:43:40,907] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:43:42,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:43:42,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:43:42,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:43:43,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:43:43,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:43:44,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:43:44,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:43:44,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:43:45,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:43:45,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:43:46,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:43:46,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:43:47,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:43:47,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:43:47,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:43:48,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:43:48,387] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:43:48,394] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf +rank0 [2026-11-15 08:43:49,622] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_qq_pf +rank0 [2026-11-15 08:43:50,039] INFO : Plotted number of elements +rank0 [2026-11-15 08:43:51,331] INFO : Plotted jets +rank0 [2026-11-15 08:43:52,362] INFO : Plotted jet ratio +rank0 [2026-11-15 08:43:52,373] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:43:53,999] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:43:54,462] INFO : Plotted number of elements +rank0 [2026-11-15 08:43:56,395] INFO : Plotted jets +rank0 [2026-11-15 08:43:57,423] INFO : Plotted jet ratio +rank0 [2026-11-15 08:43:57,432] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:43:58,926] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_83000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:43:59,370] INFO : Plotted number of elements +rank0 [2026-11-15 08:44:00,692] INFO : Plotted jets +rank0 [2026-11-15 08:44:01,680] INFO : Plotted jet ratio +rank0 [2026-11-15 08:47:55,539] INFO : Step 84000/500000 rank0 | Train Loss: 1.6607 | Train Loss Unweighted: 0.8552 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0068 | Regression_energy: 0.3005 | Regression_eta: 0.0173 | Regression_pt: 0.2906 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0885 | Classification_binary: 0.2340 | Regression_cos_phi: 0.4008 | Regression_energy: 0.1798 | Regression_eta: 0.2243 | Regression_pt: 0.1738 | Regression_sin_phi: 0.3595 | LR: 3.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.4337s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0025 | max_forward: 0.0606 | max_loss: 0.0027 | max_backward: 0.1277 | max_optimizer_step: 0.0973 | max_tensorboard_logging: 1.7163 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11356.6 | uss_MB: 10212.5 | child_rss_MB: 23800.4 | allocated_MB: 196.4 | reserved_MB: 22190.0 | inactive_split_MB: 439.6 +rank0 [2026-11-15 08:47:55,542] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-84000.pth +rank0 [2026-11-15 08:47:55,612] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 08:47:55,626] INFO : Running validation on rank0 +rank0 [2026-11-15 08:47:55,626] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:47:55,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:47:56,251] INFO : saved batch0_epoch84000.parquet +rank0 [2026-11-15 08:47:56,373] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:47:56,487] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:47:56,558] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:47:56,625] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:47:56,692] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:47:56,758] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:47:56,825] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:47:56,932] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:47:57,037] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:47:57,139] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:47:57,208] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:47:57,274] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:47:57,341] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:47:57,409] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:47:57,478] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:47:57,585] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:47:57,606] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:47:57,618] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:47:57,621] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:47:57,630] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:47:57,644] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:47:57,647] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:47:57,657] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:47:57,660] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:47:57,663] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:47:57,665] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:47:57,668] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:47:57,670] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:47:57,673] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:47:57,676] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:47:57,678] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:48:05,366] INFO : VALIDATION | Step=84000/500000 | Train Loss=1.6607 | Valid Loss=2.7099 | Train Loss Unweighted=0.8552 | Valid Loss Unweighted=0.8406 | Stale=25 | ETA=1927.2m +rank0 [2026-11-15 08:48:05,367] INFO : split_configs=['1'] +rank0 [2026-11-15 08:48:05,383] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:48:05,393] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:48:05,517] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:48:06,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:48:06,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:48:07,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:48:07,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:48:08,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:48:08,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:48:08,698] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:48:09,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:48:09,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:48:09,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:48:10,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:48:10,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:48:10,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:48:11,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:48:11,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:48:11,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:48:11,856] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:48:11,860] INFO : split_configs=['1'] +rank0 [2026-11-15 08:48:11,878] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:48:11,881] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:48:11,970] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:48:13,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:48:13,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:48:14,169] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:48:14,634] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:48:15,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:48:15,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:48:15,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:48:16,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:48:16,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:48:17,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:48:17,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:48:18,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:48:18,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:48:19,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:48:19,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:48:19,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:48:20,102] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:48:20,106] INFO : split_configs=['1'] +rank0 [2026-11-15 08:48:20,122] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:48:20,125] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:48:20,217] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:48:21,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:48:21,876] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:48:22,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:48:22,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:48:23,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:48:23,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:48:23,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:48:24,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:48:24,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:48:25,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:48:25,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:48:25,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:48:26,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:48:26,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:48:27,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:48:27,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:48:27,743] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:48:27,751] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf +rank0 [2026-11-15 08:48:28,970] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_qq_pf +rank0 [2026-11-15 08:48:29,399] INFO : Plotted number of elements +rank0 [2026-11-15 08:48:30,692] INFO : Plotted jets +rank0 [2026-11-15 08:48:31,735] INFO : Plotted jet ratio +rank0 [2026-11-15 08:48:31,747] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:48:33,378] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:48:33,819] INFO : Plotted number of elements +rank0 [2026-11-15 08:48:35,748] INFO : Plotted jets +rank0 [2026-11-15 08:48:36,795] INFO : Plotted jet ratio +rank0 [2026-11-15 08:48:36,806] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:48:38,258] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_84000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:48:38,696] INFO : Plotted number of elements +rank0 [2026-11-15 08:48:40,042] INFO : Plotted jets +rank0 [2026-11-15 08:48:41,027] INFO : Plotted jet ratio +rank0 [2026-11-15 08:48:41,527] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 08:48:41,527] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:52:36,546] INFO : Step 85000/500000 rank0 | Train Loss: 1.5298 | Train Loss Unweighted: 0.6989 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2461 | Regression_cos_phi: 0.0046 | Regression_energy: 0.2075 | Regression_eta: 0.0291 | Regression_pt: 0.2064 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.2461 | Regression_cos_phi: 0.2683 | Regression_energy: 0.1242 | Regression_eta: 0.3780 | Regression_pt: 0.1234 | Regression_sin_phi: 0.3108 | LR: 3.10e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5745s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0029 | max_forward: 0.0565 | max_loss: 0.0029 | max_backward: 0.1166 | max_optimizer_step: 0.0986 | max_tensorboard_logging: 1.8675 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 11453.0 | uss_MB: 10308.8 | child_rss_MB: 23721.8 | allocated_MB: 197.4 | reserved_MB: 18352.0 | inactive_split_MB: 236.6 +rank0 [2026-11-15 08:52:36,551] INFO : Running validation on rank0 +rank0 [2026-11-15 08:52:36,551] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:52:36,551] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:52:37,184] INFO : saved batch0_epoch85000.parquet +rank0 [2026-11-15 08:52:37,310] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:52:37,423] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:52:37,494] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:52:37,563] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:52:37,631] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:52:37,702] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:52:37,770] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:52:37,884] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:52:37,990] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:52:38,093] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:52:38,162] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:52:38,229] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:52:38,294] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:52:38,361] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:52:38,427] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:52:38,530] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:52:38,537] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:52:38,540] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:52:38,543] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:52:38,545] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:52:38,548] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:52:38,551] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:52:38,553] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:52:38,556] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:52:38,559] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:52:38,562] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:52:38,564] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:52:38,568] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:52:38,570] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:52:38,573] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:52:38,575] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:52:46,272] INFO : VALIDATION | Step=85000/500000 | Train Loss=1.5298 | Valid Loss=2.6408 | Train Loss Unweighted=0.6989 | Valid Loss Unweighted=0.8313 | Stale=26 | ETA=1922.8m +rank0 [2026-11-15 08:52:46,272] INFO : split_configs=['1'] +rank0 [2026-11-15 08:52:46,287] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:52:46,296] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:52:46,403] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:52:47,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:52:47,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:52:48,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:52:48,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:52:48,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:52:49,171] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:52:49,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:52:49,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:52:50,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:52:50,539] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:52:50,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:52:51,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:52:51,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:52:51,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:52:52,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:52:52,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:52:52,654] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:52:52,658] INFO : split_configs=['1'] +rank0 [2026-11-15 08:52:52,675] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:52:52,679] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:52:52,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:52:53,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:52:54,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:52:54,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:52:55,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:52:55,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:52:56,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:52:56,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:52:57,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:52:57,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:52:57,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:52:58,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:52:58,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:52:59,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:52:59,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:53:00,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:53:00,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:53:00,821] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:53:00,825] INFO : split_configs=['1'] +rank0 [2026-11-15 08:53:00,842] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:53:00,846] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:53:00,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:53:02,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:53:02,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:53:02,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:53:03,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:53:03,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:53:04,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:53:04,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:53:04,993] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:53:05,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:53:05,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:53:06,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:53:06,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:53:07,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:53:07,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:53:07,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:53:08,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:53:08,476] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:53:08,488] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf +rank0 [2026-11-15 08:53:09,741] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_qq_pf +rank0 [2026-11-15 08:53:10,206] INFO : Plotted number of elements +rank0 [2026-11-15 08:53:11,514] INFO : Plotted jets +rank0 [2026-11-15 08:53:12,553] INFO : Plotted jet ratio +rank0 [2026-11-15 08:53:12,566] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:53:14,327] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:53:14,773] INFO : Plotted number of elements +rank0 [2026-11-15 08:53:16,736] INFO : Plotted jets +rank0 [2026-11-15 08:53:17,796] INFO : Plotted jet ratio +rank0 [2026-11-15 08:53:17,805] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:53:19,350] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_85000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:53:19,795] INFO : Plotted number of elements +rank0 [2026-11-15 08:53:21,150] INFO : Plotted jets +rank0 [2026-11-15 08:53:22,149] INFO : Plotted jet ratio +rank0 [2026-11-15 08:55:18,507] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 08:55:18,507] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 08:57:17,610] INFO : Step 86000/500000 rank0 | Train Loss: 1.1802 | Train Loss Unweighted: 0.6757 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2322 | Regression_cos_phi: 0.0039 | Regression_energy: 0.2151 | Regression_eta: 0.0125 | Regression_pt: 0.2080 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2322 | Regression_cos_phi: 0.2279 | Regression_energy: 0.1288 | Regression_eta: 0.1622 | Regression_pt: 0.1244 | Regression_sin_phi: 0.2289 | LR: 3.15e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.6251s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0029 | max_forward: 0.0672 | max_loss: 0.0030 | max_backward: 0.1431 | max_optimizer_step: 0.0995 | max_tensorboard_logging: 1.8721 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 11364.7 | uss_MB: 10220.6 | child_rss_MB: 23725.9 | allocated_MB: 197.5 | reserved_MB: 17468.0 | inactive_split_MB: 268.5 +rank0 [2026-11-15 08:57:17,612] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-86000.pth +rank0 [2026-11-15 08:57:17,683] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 08:57:17,698] INFO : Running validation on rank0 +rank0 [2026-11-15 08:57:17,698] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 08:57:17,698] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 08:57:18,315] INFO : saved batch0_epoch86000.parquet +rank0 [2026-11-15 08:57:18,442] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 08:57:18,565] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 08:57:18,639] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 08:57:18,708] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 08:57:18,777] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 08:57:18,846] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 08:57:18,916] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 08:57:19,029] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 08:57:19,135] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 08:57:19,238] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 08:57:19,306] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 08:57:19,372] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 08:57:19,439] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 08:57:19,506] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 08:57:19,572] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 08:57:19,675] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 08:57:19,695] INFO : plotted pt_target histogram +rank0 [2026-11-15 08:57:19,708] INFO : plotted pt_pred histogram +rank0 [2026-11-15 08:57:19,711] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 08:57:19,721] INFO : plotted eta_target histogram +rank0 [2026-11-15 08:57:19,733] INFO : plotted eta_pred histogram +rank0 [2026-11-15 08:57:19,737] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 08:57:19,743] INFO : plotted sphi_target histogram +rank0 [2026-11-15 08:57:19,746] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 08:57:19,748] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 08:57:19,750] INFO : plotted cphi_target histogram +rank0 [2026-11-15 08:57:19,754] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 08:57:19,757] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 08:57:19,759] INFO : plotted energy_target histogram +rank0 [2026-11-15 08:57:19,762] INFO : plotted energy_pred histogram +rank0 [2026-11-15 08:57:19,764] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 08:57:27,436] INFO : VALIDATION | Step=86000/500000 | Train Loss=1.1802 | Valid Loss=2.6541 | Train Loss Unweighted=0.6757 | Valid Loss Unweighted=0.8425 | Stale=27 | ETA=1918.4m +rank0 [2026-11-15 08:57:27,436] INFO : split_configs=['1'] +rank0 [2026-11-15 08:57:27,457] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 08:57:27,465] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 08:57:27,580] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:57:28,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 08:57:29,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 08:57:29,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 08:57:29,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 08:57:30,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 08:57:30,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 08:57:30,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 08:57:31,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 08:57:31,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 08:57:31,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 08:57:32,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 08:57:32,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 08:57:32,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 08:57:33,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 08:57:33,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 08:57:33,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 08:57:33,965] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 08:57:33,968] INFO : split_configs=['1'] +rank0 [2026-11-15 08:57:33,986] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 08:57:33,990] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 08:57:34,080] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:57:35,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 08:57:35,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 08:57:36,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 08:57:36,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 08:57:37,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 08:57:37,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 08:57:37,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 08:57:38,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 08:57:38,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 08:57:39,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 08:57:39,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 08:57:40,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 08:57:40,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 08:57:41,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 08:57:41,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 08:57:41,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 08:57:42,120] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 08:57:42,123] INFO : split_configs=['1'] +rank0 [2026-11-15 08:57:42,141] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 08:57:42,144] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:57:42,234] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 08:57:43,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 08:57:43,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 08:57:44,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 08:57:44,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 08:57:45,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 08:57:45,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 08:57:45,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 08:57:46,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 08:57:46,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 08:57:47,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 08:57:47,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 08:57:47,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 08:57:48,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 08:57:48,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 08:57:49,209] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 08:57:49,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 08:57:49,758] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 08:57:49,767] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf +rank0 [2026-11-15 08:57:51,003] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_qq_pf +rank0 [2026-11-15 08:57:51,433] INFO : Plotted number of elements +rank0 [2026-11-15 08:57:52,742] INFO : Plotted jets +rank0 [2026-11-15 08:57:53,766] INFO : Plotted jet ratio +rank0 [2026-11-15 08:57:53,791] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:57:55,513] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ttbar_pf +rank0 [2026-11-15 08:57:55,959] INFO : Plotted number of elements +rank0 [2026-11-15 08:57:57,913] INFO : Plotted jets +rank0 [2026-11-15 08:57:58,966] INFO : Plotted jet ratio +rank0 [2026-11-15 08:57:58,981] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:58:00,471] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_86000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 08:58:00,919] INFO : Plotted number of elements +rank0 [2026-11-15 08:58:02,269] INFO : Plotted jets +rank0 [2026-11-15 08:58:03,259] INFO : Plotted jet ratio +rank0 [2026-11-15 09:01:58,239] INFO : Step 87000/500000 rank0 | Train Loss: 1.3234 | Train Loss Unweighted: 0.6338 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1992 | Regression_eta: 0.0255 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0833 | Classification_binary: 0.2101 | Regression_cos_phi: 0.1993 | Regression_energy: 0.1192 | Regression_eta: 0.3309 | Regression_pt: 0.1143 | Regression_sin_phi: 0.2663 | LR: 3.20e-04 | DataLoad Time: 0.0022s | Model Forward Time: 2.7336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0026 | max_forward: 0.0622 | max_loss: 0.0028 | max_backward: 0.1322 | max_optimizer_step: 0.0997 | max_tensorboard_logging: 1.9975 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 11484.2 | uss_MB: 10340.1 | child_rss_MB: 23794.0 | allocated_MB: 196.0 | reserved_MB: 21958.0 | inactive_split_MB: 226.0 +rank0 [2026-11-15 09:01:58,242] INFO : Running validation on rank0 +rank0 [2026-11-15 09:01:58,242] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 09:01:58,242] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 09:01:58,880] INFO : saved batch0_epoch87000.parquet +rank0 [2026-11-15 09:01:59,003] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 09:01:59,120] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 09:01:59,190] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 09:01:59,258] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 09:01:59,324] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 09:01:59,392] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 09:01:59,459] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 09:01:59,570] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 09:01:59,674] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 09:01:59,776] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 09:01:59,844] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 09:01:59,911] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 09:01:59,976] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 09:02:00,044] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 09:02:00,115] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 09:02:00,222] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 09:02:00,241] INFO : plotted pt_target histogram +rank0 [2026-11-15 09:02:00,254] INFO : plotted pt_pred histogram +rank0 [2026-11-15 09:02:00,257] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 09:02:00,272] INFO : plotted eta_target histogram +rank0 [2026-11-15 09:02:00,275] INFO : plotted eta_pred histogram +rank0 [2026-11-15 09:02:00,277] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 09:02:00,281] INFO : plotted sphi_target histogram +rank0 [2026-11-15 09:02:00,297] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 09:02:00,299] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 09:02:00,301] INFO : plotted cphi_target histogram +rank0 [2026-11-15 09:02:00,304] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 09:02:00,306] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 09:02:00,308] INFO : plotted energy_target histogram +rank0 [2026-11-15 09:02:00,311] INFO : plotted energy_pred histogram +rank0 [2026-11-15 09:02:00,313] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 09:02:08,033] INFO : VALIDATION | Step=87000/500000 | Train Loss=1.3234 | Valid Loss=2.6329 | Train Loss Unweighted=0.6338 | Valid Loss Unweighted=0.8365 | Stale=28 | ETA=1914.0m +rank0 [2026-11-15 09:02:08,033] INFO : split_configs=['1'] +rank0 [2026-11-15 09:02:08,050] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 09:02:08,057] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 09:02:08,178] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:02:09,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 09:02:09,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 09:02:10,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 09:02:10,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 09:02:10,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 09:02:11,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 09:02:11,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 09:02:11,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 09:02:12,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 09:02:12,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 09:02:12,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 09:02:13,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 09:02:13,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 09:02:13,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 09:02:14,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 09:02:14,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 09:02:14,616] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 09:02:14,619] INFO : split_configs=['1'] +rank0 [2026-11-15 09:02:14,637] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 09:02:14,641] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 09:02:14,731] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:02:16,113] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 09:02:16,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 09:02:16,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 09:02:17,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 09:02:17,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 09:02:18,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 09:02:18,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 09:02:19,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 09:02:19,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 09:02:20,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 09:02:20,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 09:02:20,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 09:02:21,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 09:02:21,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 09:02:22,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 09:02:22,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 09:02:22,869] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 09:02:22,873] INFO : split_configs=['1'] +rank0 [2026-11-15 09:02:22,890] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 09:02:22,894] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:02:22,986] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:02:24,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 09:02:24,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 09:02:25,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 09:02:25,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 09:02:25,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 09:02:26,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 09:02:26,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 09:02:27,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 09:02:27,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 09:02:27,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 09:02:28,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 09:02:28,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 09:02:29,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 09:02:29,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 09:02:29,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 09:02:30,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 09:02:30,538] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 09:02:30,547] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf +rank0 [2026-11-15 09:02:31,760] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_qq_pf +rank0 [2026-11-15 09:02:32,186] INFO : Plotted number of elements +rank0 [2026-11-15 09:02:33,479] INFO : Plotted jets +rank0 [2026-11-15 09:02:34,507] INFO : Plotted jet ratio +rank0 [2026-11-15 09:02:34,520] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:02:36,177] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:02:36,639] INFO : Plotted number of elements +rank0 [2026-11-15 09:02:38,559] INFO : Plotted jets +rank0 [2026-11-15 09:02:39,586] INFO : Plotted jet ratio +rank0 [2026-11-15 09:02:39,595] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:02:41,099] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_87000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:02:41,540] INFO : Plotted number of elements +rank0 [2026-11-15 09:02:42,851] INFO : Plotted jets +rank0 [2026-11-15 09:02:43,840] INFO : Plotted jet ratio +rank0 [2026-11-15 09:02:44,344] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 09:02:44,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 09:06:39,306] INFO : Step 88000/500000 rank0 | Train Loss: 1.2959 | Train Loss Unweighted: 0.7170 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0033 | Regression_energy: 0.2280 | Regression_eta: 0.0200 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2387 | Regression_cos_phi: 0.1959 | Regression_energy: 0.1364 | Regression_eta: 0.2604 | Regression_pt: 0.1331 | Regression_sin_phi: 0.2634 | LR: 3.25e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.7512s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0023 | max_forward: 0.0585 | max_loss: 0.0028 | max_backward: 0.1243 | max_optimizer_step: 0.0999 | max_tensorboard_logging: 1.9806 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11651.0 | uss_MB: 10506.8 | child_rss_MB: 23863.8 | allocated_MB: 193.3 | reserved_MB: 18734.0 | inactive_split_MB: 192.7 +rank0 [2026-11-15 09:06:39,309] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-88000.pth +rank0 [2026-11-15 09:06:39,382] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 09:06:39,385] INFO : Running validation on rank0 +rank0 [2026-11-15 09:06:39,385] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 09:06:39,385] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 09:06:40,017] INFO : saved batch0_epoch88000.parquet +rank0 [2026-11-15 09:06:40,147] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 09:06:40,262] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 09:06:40,333] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 09:06:40,401] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 09:06:40,467] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 09:06:40,534] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 09:06:40,602] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 09:06:40,712] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 09:06:40,818] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 09:06:40,924] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 09:06:40,995] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 09:06:41,064] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 09:06:41,134] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 09:06:41,205] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 09:06:41,274] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 09:06:41,380] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 09:06:41,387] INFO : plotted pt_target histogram +rank0 [2026-11-15 09:06:41,390] INFO : plotted pt_pred histogram +rank0 [2026-11-15 09:06:41,392] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 09:06:41,395] INFO : plotted eta_target histogram +rank0 [2026-11-15 09:06:41,398] INFO : plotted eta_pred histogram +rank0 [2026-11-15 09:06:41,400] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 09:06:41,402] INFO : plotted sphi_target histogram +rank0 [2026-11-15 09:06:41,405] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 09:06:41,407] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 09:06:41,410] INFO : plotted cphi_target histogram +rank0 [2026-11-15 09:06:41,412] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 09:06:41,415] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 09:06:41,417] INFO : plotted energy_target histogram +rank0 [2026-11-15 09:06:41,420] INFO : plotted energy_pred histogram +rank0 [2026-11-15 09:06:41,422] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 09:06:49,165] INFO : VALIDATION | Step=88000/500000 | Train Loss=1.2959 | Valid Loss=2.6176 | Train Loss Unweighted=0.7170 | Valid Loss Unweighted=0.8337 | Stale=29 | ETA=1909.6m +rank0 [2026-11-15 09:06:49,165] INFO : split_configs=['1'] +rank0 [2026-11-15 09:06:49,180] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 09:06:49,190] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 09:06:49,299] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:06:50,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 09:06:50,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 09:06:51,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 09:06:51,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 09:06:51,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 09:06:52,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 09:06:52,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 09:06:52,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 09:06:53,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 09:06:53,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 09:06:53,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 09:06:54,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 09:06:54,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 09:06:54,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 09:06:55,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 09:06:55,461] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 09:06:55,580] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 09:06:55,583] INFO : split_configs=['1'] +rank0 [2026-11-15 09:06:55,599] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 09:06:55,603] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 09:06:55,687] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:06:56,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 09:06:57,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 09:06:57,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 09:06:58,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 09:06:58,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 09:06:59,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 09:06:59,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 09:06:59,997] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 09:07:00,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 09:07:00,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 09:07:01,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 09:07:01,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 09:07:02,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 09:07:02,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 09:07:03,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 09:07:03,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 09:07:03,714] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 09:07:03,718] INFO : split_configs=['1'] +rank0 [2026-11-15 09:07:03,735] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 09:07:03,738] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:07:03,826] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:07:05,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 09:07:05,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 09:07:05,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 09:07:06,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 09:07:06,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 09:07:07,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 09:07:07,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 09:07:07,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 09:07:08,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 09:07:08,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 09:07:09,093] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 09:07:09,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 09:07:09,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 09:07:10,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 09:07:10,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 09:07:11,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 09:07:11,281] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 09:07:11,290] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf +rank0 [2026-11-15 09:07:12,537] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_qq_pf +rank0 [2026-11-15 09:07:12,976] INFO : Plotted number of elements +rank0 [2026-11-15 09:07:14,286] INFO : Plotted jets +rank0 [2026-11-15 09:07:15,311] INFO : Plotted jet ratio +rank0 [2026-11-15 09:07:15,323] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:07:17,062] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:07:17,506] INFO : Plotted number of elements +rank0 [2026-11-15 09:07:19,471] INFO : Plotted jets +rank0 [2026-11-15 09:07:20,523] INFO : Plotted jet ratio +rank0 [2026-11-15 09:07:20,533] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:07:22,021] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_88000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:07:22,468] INFO : Plotted number of elements +rank0 [2026-11-15 09:07:23,819] INFO : Plotted jets +rank0 [2026-11-15 09:07:24,821] INFO : Plotted jet ratio +rank0 [2026-11-15 09:09:21,252] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 09:09:21,252] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 09:11:19,970] INFO : Step 89000/500000 rank0 | Train Loss: 1.2001 | Train Loss Unweighted: 0.7272 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2398 | Regression_cos_phi: 0.0048 | Regression_energy: 0.2359 | Regression_eta: 0.0094 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 269.5875 | Classification_binary: 1.0000 | Regression_cos_phi: 58.7465 | Regression_energy: 0.5985 | Regression_eta: 12.9910 | Regression_pt: 0.5981 | Regression_sin_phi: 62.9006 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.2398 | Regression_cos_phi: 0.2792 | Regression_energy: 0.1412 | Regression_eta: 0.1224 | Regression_pt: 0.1398 | Regression_sin_phi: 0.2014 | LR: 3.29e-04 | DataLoad Time: 0.0023s | Model Forward Time: 2.5640s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0028 | max_forward: 0.0700 | max_loss: 0.0032 | max_backward: 0.1516 | max_optimizer_step: 0.1023 | max_tensorboard_logging: 1.7682 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 11622.6 | uss_MB: 10478.5 | child_rss_MB: 23648.5 | allocated_MB: 197.4 | reserved_MB: 23364.0 | inactive_split_MB: 496.6 +rank0 [2026-11-15 09:11:19,975] INFO : Running validation on rank0 +rank0 [2026-11-15 09:11:19,976] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 09:11:19,976] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 09:11:20,608] INFO : saved batch0_epoch89000.parquet +rank0 [2026-11-15 09:11:20,735] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 09:11:20,851] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 09:11:20,922] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 09:11:20,991] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 09:11:21,061] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 09:11:21,130] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 09:11:21,199] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 09:11:21,313] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 09:11:21,420] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 09:11:21,527] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 09:11:21,598] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 09:11:21,669] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 09:11:21,738] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 09:11:21,807] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 09:11:21,874] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 09:11:21,979] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 09:11:21,987] INFO : plotted pt_target histogram +rank0 [2026-11-15 09:11:21,989] INFO : plotted pt_pred histogram +rank0 [2026-11-15 09:11:21,992] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 09:11:21,995] INFO : plotted eta_target histogram +rank0 [2026-11-15 09:11:21,997] INFO : plotted eta_pred histogram +rank0 [2026-11-15 09:11:22,000] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 09:11:22,002] INFO : plotted sphi_target histogram +rank0 [2026-11-15 09:11:22,005] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 09:11:22,007] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 09:11:22,010] INFO : plotted cphi_target histogram +rank0 [2026-11-15 09:11:22,013] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 09:11:22,015] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 09:11:22,018] INFO : plotted energy_target histogram +rank0 [2026-11-15 09:11:22,020] INFO : plotted energy_pred histogram +rank0 [2026-11-15 09:11:22,023] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 09:11:29,749] INFO : VALIDATION | Step=89000/500000 | Train Loss=1.2001 | Valid Loss=2.6601 | Train Loss Unweighted=0.7272 | Valid Loss Unweighted=0.8352 | Stale=30 | ETA=1905.2m +rank0 [2026-11-15 09:11:29,749] INFO : split_configs=['1'] +rank0 [2026-11-15 09:11:29,765] INFO : test_dataset: idea_edm_qq_pf, 256 +rank0 [2026-11-15 09:11:29,773] INFO : Running predictions on idea_edm_qq_pf +rank0 [2026-11-15 09:11:29,912] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:11:31,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_0.parquet +rank0 [2026-11-15 09:11:31,389] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_1.parquet +rank0 [2026-11-15 09:11:31,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_2.parquet +rank0 [2026-11-15 09:11:32,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_3.parquet +rank0 [2026-11-15 09:11:32,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_4.parquet +rank0 [2026-11-15 09:11:32,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_5.parquet +rank0 [2026-11-15 09:11:33,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_6.parquet +rank0 [2026-11-15 09:11:33,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_7.parquet +rank0 [2026-11-15 09:11:33,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_8.parquet +rank0 [2026-11-15 09:11:34,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_9.parquet +rank0 [2026-11-15 09:11:34,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_10.parquet +rank0 [2026-11-15 09:11:34,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_11.parquet +rank0 [2026-11-15 09:11:35,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_12.parquet +rank0 [2026-11-15 09:11:35,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_13.parquet +rank0 [2026-11-15 09:11:35,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_14.parquet +rank0 [2026-11-15 09:11:36,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf/pred_0_15.parquet +rank0 [2026-11-15 09:11:36,275] INFO : Time taken to make predictions on device 0 is: 0.10 min +rank0 [2026-11-15 09:11:36,278] INFO : split_configs=['1'] +rank0 [2026-11-15 09:11:36,296] INFO : test_dataset: idea_edm_ttbar_pf, 256 +rank0 [2026-11-15 09:11:36,300] INFO : Running predictions on idea_edm_ttbar_pf +rank0 [2026-11-15 09:11:36,391] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:11:37,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 09:11:38,105] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 09:11:38,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 09:11:38,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 09:11:39,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 09:11:39,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 09:11:40,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 09:11:40,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 09:11:41,219] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 09:11:41,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 09:11:42,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 09:11:42,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 09:11:43,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 09:11:43,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 09:11:43,913] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 09:11:44,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 09:11:44,475] INFO : Time taken to make predictions on device 0 is: 0.13 min +rank0 [2026-11-15 09:11:44,478] INFO : split_configs=['1'] +rank0 [2026-11-15 09:11:44,496] INFO : test_dataset: idea_edm_ww_fullhad_pf, 256 +rank0 [2026-11-15 09:11:44,501] INFO : Running predictions on idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:11:44,594] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 09:11:45,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-11-15 09:11:46,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-11-15 09:11:46,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-11-15 09:11:47,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-11-15 09:11:47,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_4.parquet +rank0 [2026-11-15 09:11:47,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_5.parquet +rank0 [2026-11-15 09:11:48,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_6.parquet +rank0 [2026-11-15 09:11:48,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_7.parquet +rank0 [2026-11-15 09:11:49,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_8.parquet +rank0 [2026-11-15 09:11:49,543] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_9.parquet +rank0 [2026-11-15 09:11:49,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_10.parquet +rank0 [2026-11-15 09:11:50,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_11.parquet +rank0 [2026-11-15 09:11:50,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_12.parquet +rank0 [2026-11-15 09:11:51,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_13.parquet +rank0 [2026-11-15 09:11:51,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_14.parquet +rank0 [2026-11-15 09:11:52,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf/pred_0_15.parquet +rank0 [2026-11-15 09:11:52,169] INFO : Time taken to make predictions on device 0 is: 0.12 min +rank0 [2026-11-15 09:11:52,177] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf +rank0 [2026-11-15 09:11:53,428] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_qq_pf +rank0 [2026-11-15 09:11:53,866] INFO : Plotted number of elements +rank0 [2026-11-15 09:11:55,169] INFO : Plotted jets +rank0 [2026-11-15 09:11:56,187] INFO : Plotted jet ratio +rank0 [2026-11-15 09:11:56,198] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:11:57,875] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ttbar_pf +rank0 [2026-11-15 09:11:58,342] INFO : Plotted number of elements +rank0 [2026-11-15 09:12:00,336] INFO : Plotted jets +rank0 [2026-11-15 09:12:01,379] INFO : Plotted jet ratio +rank0 [2026-11-15 09:12:01,389] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:12:02,859] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/preds_step_89000/idea_edm_ww_fullhad_pf +rank0 [2026-11-15 09:12:03,305] INFO : Plotted number of elements +rank0 [2026-11-15 09:12:04,647] INFO : Plotted jets +rank0 [2026-11-15 09:12:05,640] INFO : Plotted jet ratio +rank0 [2026-11-15 09:12:06,156] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 09:12:06,157] INFO : Training completed. Total time on device 0: 413.163min diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..dedf0d5aac546d37c984e90ccf69782a6243b76c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f2f210c9ba4e023645c909b5b6ba965915613c8e65bb98c1f6e8aa87d1c9d93 +size 47063637 diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth new file mode 100644 index 0000000000000000000000000000000000000000..caa1353148a9abae22fa951b8b6200fbbd800d02 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:961229e606070e86f5aeac7885754f0f7a90a4bfbbafe074f5ce01c9820fdd3f +size 47102331 diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..7cccd163e506b64e91ed9e73afa9b684cc564625 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.02537974442597363, "train": {"Classification_binary": 4.014458179473877, "Classification": 0.03511136770248413, "Regression_pt": 0.24373739957809448, "Regression_eta": 0.01576177403330803, "Regression_sin_phi": 0.013775601983070374, "Regression_cos_phi": 0.014959044754505157, "Regression_energy": 0.24105654656887054, "Total": 8.90550708770752, "Total_unweighted": 4.57885991409421}, "train_group": {"cld": {"Classification_binary": 4.227148462198452, "Classification": 0.033647453105690596, "Regression_pt": 0.2955267244202648, "Regression_eta": 0.03442520246712748, "Regression_sin_phi": 0.01657513942580648, "Regression_cos_phi": 0.016763396846955853, "Regression_energy": 0.3062255900509343, "Total": 10.199777387573334, "Total_unweighted": 4.930311968515231}, "clic": {"Classification_binary": 3.9187274172499373, "Classification": 0.03680719421231174, "Regression_pt": 0.23224194397081485, "Regression_eta": 0.026012332041610825, "Regression_sin_phi": 0.018130690667655196, "Regression_cos_phi": 0.017776339423951803, "Regression_energy": 0.2362573498093688, "Total": 8.83514090558072, "Total_unweighted": 4.485953267375651}, "colliderml": {"Classification_binary": 3.21905864036835, "Classification": 0.0334811898580453, "Regression_pt": 0.43972184735017494, "Regression_eta": 0.07242583244054525, "Regression_sin_phi": 0.08091288576791952, "Regression_cos_phi": 0.08076089906978894, "Regression_energy": 0.4952729253976553, "Total": 13.617510640943372, "Total_unweighted": 4.421634220252479}}, "valid": {"Classification_binary": 3.9344515713778407, "Classification": 0.03521247993816029, "Regression_pt": 0.2922771107066761, "Regression_eta": 0.03421743349595503, "Regression_sin_phi": 0.02531306960365989, "Regression_cos_phi": 0.02580922083421187, "Regression_energy": 0.3051272999156605, "Total": 10.102435857599431, "group/cld/Classification": 0.03381000173852799, "group/cld/Classification_binary": 4.19629360767121, "group/cld/Regression_cos_phi": 0.01691305383722833, "group/cld/Regression_energy": 0.30493125509708485, "group/cld/Regression_eta": 0.03132959629626984, "group/cld/Regression_pt": 0.29340017602798785, "group/cld/Regression_sin_phi": 0.014937008949036294, "group/cld/Total": 10.10519539041722, "group/cld/Total_unweighted": 4.891614699617345, "group/clic/Classification": 0.03738528109611349, "group/clic/Classification_binary": 3.9222207576670547, "group/clic/Regression_cos_phi": 0.017077487833956455, "group/clic/Regression_energy": 0.24624213766544423, "group/clic/Regression_eta": 0.025261483293898563, "group/clic/Regression_pt": 0.2424782286299036, "group/clic/Regression_sin_phi": 0.01755539407121374, "group/clic/Total": 8.992785514669215, "group/clic/Total_unweighted": 4.5082207702575845, "group/colliderml/Classification": 0.03294967859983444, "group/colliderml/Classification_binary": 3.2012181282043457, "group/colliderml/Regression_cos_phi": 0.0775911808013916, "group/colliderml/Regression_energy": 0.47867846488952637, "group/colliderml/Regression_eta": 0.06900855898857117, "group/colliderml/Regression_pt": 0.43526241183280945, "group/colliderml/Regression_sin_phi": 0.07858095318078995, "group/colliderml/Total": 13.3539400100708, "group/colliderml/Total_unweighted": 4.373289376497269, "Total_unweighted": 4.652408185872165, "metrics/particle/class_1/efficiency": 0.8932649493217468, "metrics/particle/class_1/pid_accuracy": 0.9699721336364746, "metrics/particle/class_2/efficiency": 0.13476070761680603, "metrics/particle/class_2/pid_accuracy": 0.1743474006652832, "metrics/particle/class_3/efficiency": 0.4322757422924042, "metrics/particle/class_3/pid_accuracy": 0.9430528879165649, "metrics/particle/class_4/efficiency": 0.8355945944786072, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 0.9552669525146484, "metrics/particle/class_5/pid_accuracy": 0.003021148033440113, "metrics/particle/count/bias_mean": -4.718857288360596, "metrics/particle/count/mae": 8.814857482910156, "metrics/particle/count/prediction_mean": 87.4354248046875, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.14522570371627808, "metrics/particle/event/energy_response_mean": 1.008689045906067, "metrics/particle/event/met_abs_error": 24.388774871826172, "metrics/particle/event/scalar_pt_relative_abs_error": 0.12969157099723816, "metrics/particle/event/scalar_pt_response_mean": 1.0263148546218872, "metrics/particle/event/vector_pt_closure": 0.13720087707042694, "metrics/particle/matched/delta_eta_abs_mean": 0.02945369854569435, "metrics/particle/matched/delta_phi_abs_mean": 0.021780794486403465, "metrics/particle/matched/delta_r_mean": 0.04056001827120781, "metrics/particle/matched/energy_relative_abs_mean": 0.10659940540790558, "metrics/particle/matched/pid_accuracy": 0.9169890880584717, "metrics/particle/matched/pt_relative_abs_mean": 0.10163061320781708, "metrics/particle/matching/duplicate_fraction": 0.048806630074977875, "metrics/particle/matching/efficiency": 0.5801823139190674, "metrics/particle/matching/f1": 0.5954270362854004, "metrics/particle/matching/purity": 0.6114945411682129, "metrics/particle/matching/target_coverage_dr0p05": 0.4156135618686676, "metrics/particle/matching/target_coverage_dr0p10": 0.6195510625839233, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.536708652973175, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6988218426704407}, "step_train_time": 7.213911294937134, "step_valid_time": 11.966867106966674, "step_total_time": 1196.6680104732513, "elapsed_training_loop_seconds": 1196.6680104732513, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0041320662385482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25304969891150275, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1306, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1184, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8788694481830417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8753351206434317, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8770987239758227, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12466487935656836, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7967698519515478, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7935656836461126, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7951645399597045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8788694481830417, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9935263395904617, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22528946182192877, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1116, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 896, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 841, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8014311270125224, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8028673835125448, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8021486123545211, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1971326164874552, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7522361359570662, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.753584229390681, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7529095792300806, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8014311270125224, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.995329889950736, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21432962770809671, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 635, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 433, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 408, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.6894904458598726, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.6818897637795276, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.6856690419635788, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.31811023622047246, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6496815286624203, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6425196850393701, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6460807600950119, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.6894904458598726, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.982051462946415, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1964407937431727, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1502, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1311, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1226, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.878686327077748, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8728362183754993, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.875751503006012, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12716378162450068, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.82171581769437, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8162450066577897, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8189712758851035, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.878686327077748, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.975997838458207, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2127946356736483, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1086, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 902, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 870, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8118811881188119, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8305709023941068, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8211197086936732, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1694290976058932, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.783078307830783, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8011049723756906, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7919890760127446, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8118811881188119, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9761635689633988, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21466251247699408, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 644, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 472, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 456, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7421383647798742, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7329192546583851, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7375, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2670807453416149, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7169811320754716, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7080745341614907, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7421383647798742, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0275988672876049, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3336069766874604, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3533, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2393, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2183, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.6746546377220186, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.6773280498160203, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.6759887005649717, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.3226719501839796, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6154496757823513, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6178884800452873, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6166666666666667, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.6746546377220186, "step_test_time": 33.84841241897084, "step_plot_time": 26.68587251100689, "step_validation_cycle_time": 72.96782449295279} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_100000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_100000.json new file mode 100644 index 0000000000000000000000000000000000000000..e48ffdba31924f8ba577dfb6ff9d39bdc5c2e1d7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_100000.json @@ -0,0 +1 @@ +{"epoch": 0.2537974442597363, "train": {"Classification_binary": 3.0544657707214355, "Classification": 0.026462815701961517, "Regression_pt": 0.1773890107870102, "Regression_eta": 0.015215209685266018, "Regression_sin_phi": 0.00800138246268034, "Regression_cos_phi": 0.016383370384573936, "Regression_energy": 0.17530816793441772, "Total": 6.688070297241211, "Total_unweighted": 3.4732257276773453}, "train_group": {"cld": {"Classification_binary": 3.1136537626118, "Classification": 0.02228944250633438, "Regression_pt": 0.1901210350190808, "Regression_eta": 0.019018620685506946, "Regression_sin_phi": 0.00947069412398481, "Regression_cos_phi": 0.009602672555679065, "Regression_energy": 0.19710659659551288, "Total": 6.918337198074706, "Total_unweighted": 3.5612628240978994}, "clic": {"Classification_binary": 2.8696119047857978, "Classification": 0.02495503653254774, "Regression_pt": 0.1579453219119851, "Regression_eta": 0.013103750344175656, "Regression_sin_phi": 0.011270852180550227, "Regression_cos_phi": 0.01110205977704857, "Regression_energy": 0.16039833021504027, "Total": 6.1586655739907386, "Total_unweighted": 3.2483872557471454}, "colliderml": {"Classification_binary": 2.605635333705593, "Classification": 0.024692066466888866, "Regression_pt": 0.32882315690095004, "Regression_eta": 0.06205554870334831, "Regression_sin_phi": 0.06911486068421656, "Regression_cos_phi": 0.06910114733127502, "Regression_energy": 0.37101173409828553, "Total": 10.636576638207421, "Total_unweighted": 3.5304338478905573}}, "valid": {"Classification_binary": 2.9275548761541192, "Classification": 0.023772517117587003, "Regression_pt": 0.19661742123690518, "Regression_eta": 0.02158684947273948, "Regression_sin_phi": 0.017956692522222345, "Regression_cos_phi": 0.018335819244384766, "Regression_energy": 0.20632686614990234, "Total": 7.099441805752841, "group/cld/Classification": 0.021843090970465478, "group/cld/Classification_binary": 3.0791587017952127, "group/cld/Regression_cos_phi": 0.009604187721901752, "group/cld/Regression_energy": 0.19406641290542928, "group/cld/Regression_eta": 0.016435462109586026, "group/cld/Regression_pt": 0.18802813266186005, "group/cld/Regression_sin_phi": 0.008554751568652213, "group/cld/Total": 6.810933214552859, "group/cld/Total_unweighted": 3.517690739733108, "group/clic/Classification": 0.025464763032629133, "group/clic/Classification_binary": 2.886513243330286, "group/clic/Regression_cos_phi": 0.010449883785653622, "group/clic/Regression_energy": 0.1649794375642817, "group/clic/Regression_eta": 0.013608304744071148, "group/clic/Regression_pt": 0.16044018116403133, "group/clic/Regression_sin_phi": 0.010399845052272715, "group/clic/Total": 6.221485543758311, "group/clic/Total_unweighted": 3.2718556586732253, "group/colliderml/Classification": 0.02446918934583664, "group/colliderml/Classification_binary": 2.6027770042419434, "group/colliderml/Regression_cos_phi": 0.06714992225170135, "group/colliderml/Regression_energy": 0.3638003170490265, "group/colliderml/Regression_eta": 0.06015597656369209, "group/colliderml/Regression_pt": 0.3281189203262329, "group/colliderml/Regression_sin_phi": 0.06777314096689224, "group/colliderml/Total": 10.525933265686035, "group/colliderml/Total_unweighted": 3.514244470745325, "Total_unweighted": 3.4121510418978604, "metrics/particle/class_1/efficiency": 0.9435113668441772, "metrics/particle/class_1/pid_accuracy": 0.9808236956596375, "metrics/particle/class_2/efficiency": 0.2237904965877533, "metrics/particle/class_2/pid_accuracy": 0.5554046034812927, "metrics/particle/class_3/efficiency": 0.6100702881813049, "metrics/particle/class_3/pid_accuracy": 0.9692674279212952, "metrics/particle/class_4/efficiency": 0.8914604783058167, "metrics/particle/class_4/pid_accuracy": 0.18889883160591125, "metrics/particle/class_5/efficiency": 0.9783549904823303, "metrics/particle/class_5/pid_accuracy": 0.501474916934967, "metrics/particle/count/bias_mean": -0.5937142968177795, "metrics/particle/count/mae": 5.113714218139648, "metrics/particle/count/prediction_mean": 91.5605697631836, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.06347537785768509, "metrics/particle/event/energy_response_mean": 0.9778225421905518, "metrics/particle/event/met_abs_error": 7.970993995666504, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05215930566191673, "metrics/particle/event/scalar_pt_response_mean": 0.9871905446052551, "metrics/particle/event/vector_pt_closure": 0.056245025247335434, "metrics/particle/matched/delta_eta_abs_mean": 0.01970168761909008, "metrics/particle/matched/delta_phi_abs_mean": 0.01613108441233635, "metrics/particle/matched/delta_r_mean": 0.028352905064821243, "metrics/particle/matched/energy_relative_abs_mean": 0.09435772895812988, "metrics/particle/matched/pid_accuracy": 0.9458452463150024, "metrics/particle/matched/pt_relative_abs_mean": 0.088955819606781, "metrics/particle/matching/duplicate_fraction": 0.03681559860706329, "metrics/particle/matching/efficiency": 0.6925032734870911, "metrics/particle/matching/f1": 0.6947412490844727, "metrics/particle/matching/purity": 0.6969937086105347, "metrics/particle/matching/target_coverage_dr0p05": 0.5879766941070557, "metrics/particle/matching/target_coverage_dr0p10": 0.726204514503479, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6237179636955261, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7857630252838135}, "step_train_time": 7.341254711151123, "step_valid_time": 12.92231062008068, "step_total_time": 12547.073516845703, "elapsed_training_loop_seconds": 12547.073516845703, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974772314910045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1222140202662112, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1390, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1378, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9353970390309556, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9385550303848751, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9369733737782272, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06144496961512492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.927321668909825, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9304523970290345, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9288843950117964, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9353970390309556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9903662708945717, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13257207697152806, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1109, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1020, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1013, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9123434704830053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9197475202885482, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.916030534351145, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08025247971145176, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.90608228980322, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9134355275022543, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9097440502918724, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9123434704830053, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9949312187708907, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1268595121917534, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 533, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 532, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8487261146496815, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8514376996805112, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8500797448165869, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1485623003194888, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8471337579617835, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8498402555910544, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8484848484848485, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8487261146496815, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9957090408523506, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07891544035093068, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1491, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1394, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9376675603217158, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9382964453386988, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.937981897418706, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06170355466130114, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9343163538873994, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9349429912810194, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9346295675494468, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9376675603217158, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9906357206405881, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0941039325568852, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1099, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 997, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 996, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8973897389738974, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9071883530482256, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.902262443438914, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09281164695177434, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8964896489648965, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9062784349408554, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9013574660633484, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8973897389738974, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9912776370885132, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10166094956381722, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 634, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 550, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 548, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8647798742138365, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8675078864353313, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8661417322834646, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13249211356466878, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8616352201257862, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8643533123028391, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.862992125984252, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8647798742138365, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9707062595476686, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1788899278665489, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3603, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2638, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2594, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7437270933182971, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7321676380793783, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7379020979020979, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2678323619206217, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7313222441499859, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7199555925617541, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7255944055944056, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7437270933182971, "step_test_time": 34.191106230951846, "step_plot_time": 26.714264777023345, "step_validation_cycle_time": 74.33197715494316} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_110000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_110000.json new file mode 100644 index 0000000000000000000000000000000000000000..735d6c76abab1d316b273e6398232ce846ca138f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_110000.json @@ -0,0 +1 @@ +{"epoch": 0.27917718868570995, "train": {"Classification_binary": 2.8021388053894043, "Classification": 0.01900961808860302, "Regression_pt": 0.13311579823493958, "Regression_eta": 0.010256677865982056, "Regression_sin_phi": 0.009020792320370674, "Regression_cos_phi": 0.0087897889316082, "Regression_energy": 0.12820445001125336, "Total": 5.484050750732422, "Total_unweighted": 3.110535930842161}, "train_group": {"cld": {"Classification_binary": 2.9703322265957213, "Classification": 0.021020961407985952, "Regression_pt": 0.1786565837693644, "Regression_eta": 0.017993620870908996, "Regression_sin_phi": 0.008936439848479789, "Regression_cos_phi": 0.009095703502669483, "Regression_energy": 0.18600904138328078, "Total": 6.553775668502212, "Total_unweighted": 3.3920445773784107}, "clic": {"Classification_binary": 2.768687558745196, "Classification": 0.02318667874442871, "Regression_pt": 0.15202899736737063, "Regression_eta": 0.012779463343266509, "Regression_sin_phi": 0.010624893875458015, "Regression_cos_phi": 0.010633536001031627, "Regression_energy": 0.1535419286368136, "Total": 5.9153403921755485, "Total_unweighted": 3.1314830567135647}, "colliderml": {"Classification_binary": 2.5581667050943, "Classification": 0.02423197228472691, "Regression_pt": 0.32157680441488373, "Regression_eta": 0.06128522598215409, "Regression_sin_phi": 0.06852272334131035, "Regression_cos_phi": 0.06850069300369457, "Regression_energy": 0.36262690134950587, "Total": 10.437790246339175, "Total_unweighted": 3.4649110254705757}}, "valid": {"Classification_binary": 2.8151505903764202, "Classification": 0.022733142159201882, "Regression_pt": 0.19023238095370207, "Regression_eta": 0.020794042673977938, "Regression_sin_phi": 0.01757828105579723, "Regression_cos_phi": 0.0177853909405795, "Regression_energy": 0.19886613325639205, "Total": 6.845487837357955, "group/cld/Classification": 0.020850346443500926, "group/cld/Classification_binary": 2.947255236037234, "group/cld/Regression_cos_phi": 0.009127013860864843, "group/cld/Regression_energy": 0.18762513424487823, "group/cld/Regression_eta": 0.01571618749740276, "group/cld/Regression_pt": 0.1821781726593667, "group/cld/Regression_sin_phi": 0.008381557591418003, "group/cld/Total": 6.553855084358378, "group/cld/Total_unweighted": 3.371133648334665, "group/clic/Classification": 0.02419899118707535, "group/clic/Classification_binary": 2.7723067263339427, "group/clic/Regression_cos_phi": 0.00990180512692066, "group/clic/Regression_energy": 0.15824201259207218, "group/clic/Regression_eta": 0.012716212171189328, "group/clic/Regression_pt": 0.1545985404481279, "group/clic/Regression_sin_phi": 0.009924112482273833, "group/clic/Total": 5.97099466526762, "group/clic/Total_unweighted": 3.1418884003416023, "group/colliderml/Classification": 0.023957934230566025, "group/colliderml/Classification_binary": 2.5529510974884033, "group/colliderml/Regression_cos_phi": 0.0663774162530899, "group/colliderml/Regression_energy": 0.35121989250183105, "group/colliderml/Regression_eta": 0.05943884700536728, "group/colliderml/Regression_pt": 0.31856584548950195, "group/colliderml/Regression_sin_phi": 0.06707777827978134, "group/colliderml/Total": 10.27098560333252, "group/colliderml/Total_unweighted": 3.439588811248541, "Total_unweighted": 3.2831399614160706, "metrics/particle/class_1/efficiency": 0.9434804916381836, "metrics/particle/class_1/pid_accuracy": 0.9777770638465881, "metrics/particle/class_2/efficiency": 0.24454964697360992, "metrics/particle/class_2/pid_accuracy": 0.6002486348152161, "metrics/particle/class_3/efficiency": 0.6342540383338928, "metrics/particle/class_3/pid_accuracy": 0.9703951478004456, "metrics/particle/class_4/efficiency": 0.883479654788971, "metrics/particle/class_4/pid_accuracy": 0.26829269528388977, "metrics/particle/class_5/efficiency": 0.9696969985961914, "metrics/particle/class_5/pid_accuracy": 0.6116071343421936, "metrics/particle/count/bias_mean": 0.8074285984039307, "metrics/particle/count/mae": 4.8874287605285645, "metrics/particle/count/prediction_mean": 92.96171569824219, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.058047764003276825, "metrics/particle/event/energy_response_mean": 1.0089762210845947, "metrics/particle/event/met_abs_error": 7.018613338470459, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04687224328517914, "metrics/particle/event/scalar_pt_response_mean": 1.0141733884811401, "metrics/particle/event/vector_pt_closure": 0.051727812737226486, "metrics/particle/matched/delta_eta_abs_mean": 0.01591651327908039, "metrics/particle/matched/delta_phi_abs_mean": 0.01520993560552597, "metrics/particle/matched/delta_r_mean": 0.024584440514445305, "metrics/particle/matched/energy_relative_abs_mean": 0.09569159150123596, "metrics/particle/matched/pid_accuracy": 0.9471068382263184, "metrics/particle/matched/pt_relative_abs_mean": 0.09196917712688446, "metrics/particle/matching/duplicate_fraction": 0.03919278457760811, "metrics/particle/matching/efficiency": 0.7060891389846802, "metrics/particle/matching/f1": 0.7030093669891357, "metrics/particle/matching/purity": 0.6999563574790955, "metrics/particle/matching/target_coverage_dr0p05": 0.6267191767692566, "metrics/particle/matching/target_coverage_dr0p10": 0.7434550523757935, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6286414265632629, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7936813831329346}, "step_train_time": 7.346325635910034, "step_valid_time": 12.88495455798693, "step_total_time": 13819.086884021759, "elapsed_training_loop_seconds": 13819.086884021759, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0064649501282414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10974982907788988, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1404, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9508748317631225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547297297297297, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04527027027027027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9448183041722745, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9486486486486486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9467296021577882, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9508748317631225, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0095602008162994, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10883094037851948, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1040, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1035, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9302325581395349, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9310653536257834, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.930648769574944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06893464637421665, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9257602862254025, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9265890778871978, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9261744966442953, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9302325581395349, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0189403617372672, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1241646300159781, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 558, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 553, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8885350318471338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8942307692307693, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8913738019169329, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10576923076923077, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8805732484076433, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8862179487179487, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8833865814696485, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8885350318471338, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0054316549553644, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07477879613446947, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1410, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1404, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9450402144772118, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9418837675350702, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9434593509534962, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05811623246492986, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9410187667560321, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9378757515030061, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9394446303111409, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9450402144772118, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0081080434321636, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09935674769358926, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1100, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1026, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1021, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9234923492349235, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9327272727272727, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9280868385345997, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06727272727272728, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.918991899189919, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9281818181818182, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9235639981908639, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9234923492349235, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0063915797653051, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0984466010704308, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 641, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 565, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 561, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8883647798742138, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8814352574102964, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.884886452623336, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11856474258970359, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8820754716981132, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.875195007800312, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8786217697729053, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8883647798742138, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.992104603559367, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16712202595238757, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3619, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2697, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2654, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7603608683394418, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.745233489914341, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7527211833658946, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.25476651008565904, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7482379475613194, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7333517546283503, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7407200669829752, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7603608683394418, "step_test_time": 34.14280302100815, "step_plot_time": 26.101116303005256, "step_validation_cycle_time": 73.71217459894251} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_120000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_120000.json new file mode 100644 index 0000000000000000000000000000000000000000..48ed62b0eadc85b195164df0c12a3fda2ed8c9d5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_120000.json @@ -0,0 +1 @@ +{"epoch": 0.3045569331116836, "train": {"Classification_binary": 2.4982972145080566, "Classification": 0.024165524169802666, "Regression_pt": 0.2997454106807709, "Regression_eta": 0.058045171201229095, "Regression_sin_phi": 0.06698617339134216, "Regression_cos_phi": 0.06636327505111694, "Regression_energy": 0.3345666229724884, "Total": 9.915374755859375, "Total_unweighted": 3.348169391974807}, "train_group": {"cld": {"Classification_binary": 2.8736422555940644, "Classification": 0.02019493612698517, "Regression_pt": 0.1746241800240926, "Regression_eta": 0.016657922553966892, "Regression_sin_phi": 0.0086480609250364, "Regression_cos_phi": 0.008820771072591256, "Regression_energy": 0.18056029450517516, "Total": 6.355940821650508, "Total_unweighted": 3.283148420801912}, "clic": {"Classification_binary": 2.667159979049866, "Classification": 0.022366467418635752, "Regression_pt": 0.14417997481407704, "Regression_eta": 0.012576454579158946, "Regression_sin_phi": 0.01042391773727205, "Regression_cos_phi": 0.010229600536813205, "Regression_energy": 0.1454503403322117, "Total": 5.664053954161681, "Total_unweighted": 3.0123867344680346}, "colliderml": {"Classification_binary": 2.5111129162554255, "Classification": 0.023722195016320596, "Regression_pt": 0.31383025744361076, "Regression_eta": 0.06067324743165584, "Regression_sin_phi": 0.06798713558359061, "Regression_cos_phi": 0.06811320963510853, "Regression_energy": 0.3537218707228849, "Total": 10.234574389314938, "Total_unweighted": 3.399160832088597}}, "valid": {"Classification_binary": 2.739590731534091, "Classification": 0.021805414286526766, "Regression_pt": 0.1846880305897106, "Regression_eta": 0.02151270779696378, "Regression_sin_phi": 0.01744152849370783, "Regression_cos_phi": 0.01755370551889593, "Regression_energy": 0.19348387284712357, "Total": 6.667372270063921, "group/cld/Classification": 0.020067968267075558, "group/cld/Classification_binary": 2.8637370657413563, "group/cld/Regression_cos_phi": 0.008971959986585253, "group/cld/Regression_energy": 0.18207000164275475, "group/cld/Regression_eta": 0.016429755281894765, "group/cld/Regression_pt": 0.1765835538823554, "group/cld/Regression_sin_phi": 0.008297295012372606, "group/cld/Total": 6.369649034865359, "group/cld/Total_unweighted": 3.2761575998143946, "group/clic/Classification": 0.0229682668726495, "group/clic/Classification_binary": 2.6949832997423537, "group/clic/Regression_cos_phi": 0.0096289483790702, "group/clic/Regression_energy": 0.15294621853118248, "group/clic/Regression_eta": 0.013801548075168691, "group/clic/Regression_pt": 0.14868268560855946, "group/clic/Regression_sin_phi": 0.009776816089102563, "group/clic/Total": 5.785451036818484, "group/clic/Total_unweighted": 3.0527877832980863, "group/colliderml/Classification": 0.023493288084864616, "group/colliderml/Classification_binary": 2.5059401988983154, "group/colliderml/Regression_cos_phi": 0.06604155153036118, "group/colliderml/Regression_energy": 0.3460915684700012, "group/colliderml/Regression_eta": 0.05909539386630058, "group/colliderml/Regression_pt": 0.3142606019973755, "group/colliderml/Regression_sin_phi": 0.06681783497333527, "group/colliderml/Total": 10.132575035095215, "group/colliderml/Total_unweighted": 3.381740437820554, "Total_unweighted": 3.1960759910670196, "metrics/particle/class_1/efficiency": 0.9461689591407776, "metrics/particle/class_1/pid_accuracy": 0.9815634489059448, "metrics/particle/class_2/efficiency": 0.25110745429992676, "metrics/particle/class_2/pid_accuracy": 0.6252161860466003, "metrics/particle/class_3/efficiency": 0.6172233819961548, "metrics/particle/class_3/pid_accuracy": 0.970551609992981, "metrics/particle/class_4/efficiency": 0.8978451490402222, "metrics/particle/class_4/pid_accuracy": 0.24888889491558075, "metrics/particle/class_5/efficiency": 0.9754689931869507, "metrics/particle/class_5/pid_accuracy": 0.5680473446846008, "metrics/particle/count/bias_mean": -1.3857142925262451, "metrics/particle/count/mae": 5.157142639160156, "metrics/particle/count/prediction_mean": 90.76856994628906, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.05884037911891937, "metrics/particle/event/energy_response_mean": 0.9968576431274414, "metrics/particle/event/met_abs_error": 7.35196590423584, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04584818705916405, "metrics/particle/event/scalar_pt_response_mean": 1.0078426599502563, "metrics/particle/event/vector_pt_closure": 0.05064466595649719, "metrics/particle/matched/delta_eta_abs_mean": 0.016903236508369446, "metrics/particle/matched/delta_phi_abs_mean": 0.01547953486442566, "metrics/particle/matched/delta_r_mean": 0.025500968098640442, "metrics/particle/matched/energy_relative_abs_mean": 0.09433890879154205, "metrics/particle/matched/pid_accuracy": 0.949256181716919, "metrics/particle/matched/pt_relative_abs_mean": 0.0883474126458168, "metrics/particle/matching/duplicate_fraction": 0.03809373825788498, "metrics/particle/matching/efficiency": 0.7006821036338806, "metrics/particle/matching/f1": 0.7059900164604187, "metrics/particle/matching/purity": 0.7113789916038513, "metrics/particle/matching/target_coverage_dr0p05": 0.6144416332244873, "metrics/particle/matching/target_coverage_dr0p10": 0.7376511693000793, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6269423961639404, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7871705889701843}, "step_train_time": 13.169210433959961, "step_valid_time": 13.0599671730306, "step_total_time": 15095.481820583344, "elapsed_training_loop_seconds": 15095.481820583344, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0140459956087766, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10217676103850304, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9508748317631225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547297297297297, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04527027027027027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9434724091520862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9472972972972973, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9453809844908968, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9508748317631225, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.010315194261423, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10880284120564077, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1112, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1043, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1033, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9329159212880143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9379496402877698, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9354260089686098, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06205035971223022, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9239713774597496, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9289568345323741, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9264573991031391, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9329159212880143, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0179647932747804, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11346098927829085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 622, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 554, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 550, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8821656050955414, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8906752411575563, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8864, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10932475884244373, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8757961783439491, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8842443729903537, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.88, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8821656050955414, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0052424360798924, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07028018073644537, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1505, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1414, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9504021447721179, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9421926910299003, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9462796129462796, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05780730897009967, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9477211796246648, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9395348837209302, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9436102769436103, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9504021447721179, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036047451078665, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08539614351693492, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1101, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1030, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1029, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9270927092709271, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.935513169845595, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9312839059674503, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06448683015440508, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9261926192619262, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9346049046321526, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.930379746835443, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9270927092709271, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0087190367797356, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0940684700680019, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 639, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 567, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 565, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8915094339622641, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8873239436619719, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8894117647058823, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11267605633802817, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8883647798742138, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8841940532081377, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8862745098039215, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8915094339622641, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9895603972372673, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15998841453310453, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3577, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2632, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2604, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7420355229771638, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.735812133072407, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7389107243121842, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.26418786692759294, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7341415280518748, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7279843444227005, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7310499719258844, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7420355229771638, "step_test_time": 34.87460779096, "step_plot_time": 26.856718206079677, "step_validation_cycle_time": 75.40527345601004} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_130000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_130000.json new file mode 100644 index 0000000000000000000000000000000000000000..55c77b5810895f9c1a15d50322f54460f42750e9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_130000.json @@ -0,0 +1 @@ +{"epoch": 0.3299366775376572, "train": {"Classification_binary": 2.887977123260498, "Classification": 0.018218910321593285, "Regression_pt": 0.13632020354270935, "Regression_eta": 0.010815348476171494, "Regression_sin_phi": 0.010501840151846409, "Regression_cos_phi": 0.008276575244963169, "Regression_energy": 0.1372874677181244, "Total": 5.665372371673584, "Total_unweighted": 3.209397468715906}, "train_group": {"cld": {"Classification_binary": 2.8344407331443833, "Classification": 0.019950929761468294, "Regression_pt": 0.16775889177165346, "Regression_eta": 0.016720676325501393, "Regression_sin_phi": 0.008803207855978367, "Regression_cos_phi": 0.008765763980280719, "Regression_energy": 0.1740621057991496, "Total": 6.204947211785231, "Total_unweighted": 3.2305023086384153}, "clic": {"Classification_binary": 2.6184201083025775, "Classification": 0.021671968277308856, "Regression_pt": 0.14502556825632806, "Regression_eta": 0.012197995405258224, "Regression_sin_phi": 0.0102300966303941, "Regression_cos_phi": 0.010195363873011759, "Regression_energy": 0.14621696950079086, "Total": 5.614250701468986, "Total_unweighted": 2.963958070245669}, "colliderml": {"Classification_binary": 2.480170128939746, "Classification": 0.023333495712137078, "Regression_pt": 0.31042043600712454, "Regression_eta": 0.06063514463491626, "Regression_sin_phi": 0.06788548465829354, "Regression_cos_phi": 0.06794337596039514, "Regression_energy": 0.35123128978697743, "Total": 10.145000065411175, "Total_unweighted": 3.36161935569959}}, "valid": {"Classification_binary": 2.6807292591441763, "Classification": 0.021511103890158912, "Regression_pt": 0.1808396426114169, "Regression_eta": 0.020497475970875135, "Regression_sin_phi": 0.01724469553340565, "Regression_cos_phi": 0.01743124940178611, "Regression_energy": 0.18902615633877842, "Total": 6.5276561390269885, "group/cld/Classification": 0.019661137398253096, "group/cld/Classification_binary": 2.788028473549701, "group/cld/Regression_cos_phi": 0.008921720880143185, "group/cld/Regression_energy": 0.17437178023318026, "group/cld/Regression_eta": 0.015678389275327644, "group/cld/Regression_pt": 0.16970103852292326, "group/cld/Regression_sin_phi": 0.008126082572531192, "group/cld/Total": 6.1632209940159575, "group/cld/Total_unweighted": 3.1844886224320597, "group/clic/Classification": 0.02277250239189635, "group/clic/Classification_binary": 2.6462347152385304, "group/clic/Regression_cos_phi": 0.009554206056797758, "group/clic/Regression_energy": 0.15162985375586976, "group/clic/Regression_eta": 0.012309055379096498, "group/clic/Regression_pt": 0.14803885399027072, "group/clic/Regression_sin_phi": 0.009561956562894457, "group/clic/Total": 5.706821522814162, "group/clic/Total_unweighted": 3.000101143375356, "group/colliderml/Classification": 0.023240063339471817, "group/colliderml/Classification_binary": 2.466867685317993, "group/colliderml/Regression_cos_phi": 0.06556682288646698, "group/colliderml/Regression_energy": 0.3419250249862671, "group/colliderml/Regression_eta": 0.058707013726234436, "group/colliderml/Regression_pt": 0.30991145968437195, "group/colliderml/Regression_sin_phi": 0.06659866869449615, "group/colliderml/Total": 10.009385108947754, "group/colliderml/Total_unweighted": 3.3328167386353016, "Total_unweighted": 3.1272795828905973, "metrics/particle/class_1/efficiency": 0.948563814163208, "metrics/particle/class_1/pid_accuracy": 0.9812353253364563, "metrics/particle/class_2/efficiency": 0.25097715854644775, "metrics/particle/class_2/pid_accuracy": 0.597335159778595, "metrics/particle/class_3/efficiency": 0.6330245733261108, "metrics/particle/class_3/pid_accuracy": 0.9738909602165222, "metrics/particle/class_4/efficiency": 0.8914604783058167, "metrics/particle/class_4/pid_accuracy": 0.29543420672416687, "metrics/particle/class_5/efficiency": 0.9696969985961914, "metrics/particle/class_5/pid_accuracy": 0.5223214030265808, "metrics/particle/count/bias_mean": -0.673714280128479, "metrics/particle/count/mae": 4.727428436279297, "metrics/particle/count/prediction_mean": 91.4805679321289, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.05463261157274246, "metrics/particle/event/energy_response_mean": 0.992132306098938, "metrics/particle/event/met_abs_error": 6.328303337097168, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04267891123890877, "metrics/particle/event/scalar_pt_response_mean": 0.9963604807853699, "metrics/particle/event/vector_pt_closure": 0.04555545747280121, "metrics/particle/matched/delta_eta_abs_mean": 0.016219627112150192, "metrics/particle/matched/delta_phi_abs_mean": 0.015545767731964588, "metrics/particle/matched/delta_r_mean": 0.02512933686375618, "metrics/particle/matched/energy_relative_abs_mean": 0.09318207949399948, "metrics/particle/matched/pid_accuracy": 0.9495055675506592, "metrics/particle/matched/pt_relative_abs_mean": 0.08788905292749405, "metrics/particle/matching/duplicate_fraction": 0.03486142307519913, "metrics/particle/matching/efficiency": 0.7085632681846619, "metrics/particle/matching/f1": 0.7111628651618958, "metrics/particle/matching/purity": 0.7137815356254578, "metrics/particle/matching/target_coverage_dr0p05": 0.6198176741600037, "metrics/particle/matching/target_coverage_dr0p10": 0.7444409728050232, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6351770162582397, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7963849306106567}, "step_train_time": 7.354875802993774, "step_valid_time": 13.060698407003656, "step_total_time": 16365.943361520767, "elapsed_training_loop_seconds": 16365.943361520767, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0037723657284547, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0878162397528407, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1409, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1398, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9481830417227456, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501011463250169, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9491411249578983, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049898853674983146, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9407806191117093, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.942683749157114, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9417312226338834, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9481830417227456, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0000249663925795, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10397600988353539, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1044, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9418604651162791, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9427036705461056, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9422818791946309, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05729632945389436, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9338103756708408, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9346463742166518, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.934228187919463, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9418604651162791, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036627695339506, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10244936444350428, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 546, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 546, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8694267515923567, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8736, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8715083798882681, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1264, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8694267515923567, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8736, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8715083798882681, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8694267515923567, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004347583653666, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0670670210533113, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1423, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1420, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9537533512064343, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9505678022712091, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9521579123452659, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049432197728790914, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9517426273458445, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.948563794255177, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9501505520240883, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9537533512064343, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029335408003, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07972081194249314, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1101, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1029, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1026, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9261926192619262, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9346049046321526, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.930379746835443, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0653950953678474, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9234923492349235, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9318801089918256, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9276672694394213, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9261926192619262, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020909950349361, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08310808305757544, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 570, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 568, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8962264150943396, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8962264150943396, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8962264150943396, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10377358490566038, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8930817610062893, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8930817610062893, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8930817610062893, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8962264150943396, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9813412218912378, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15750182906583154, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3633, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2646, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2603, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7459825204398083, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7283236994219653, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7370473537604457, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.27167630057803466, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7338595996616859, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7164877511698321, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7250696378830084, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7459825204398083, "step_test_time": 34.53201342502143, "step_plot_time": 26.310126402997412, "step_validation_cycle_time": 74.50830869202036} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_140000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_140000.json new file mode 100644 index 0000000000000000000000000000000000000000..d7c230ed1fd672cc2268b19b9a4b8bc968fa3537 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_140000.json @@ -0,0 +1 @@ +{"epoch": 0.35531642196363084, "train": {"Classification_binary": 2.301196813583374, "Classification": 0.02416069246828556, "Regression_pt": 0.14404939115047455, "Regression_eta": 0.011342788115143776, "Regression_sin_phi": 0.010394523851573467, "Regression_cos_phi": 0.009642371907830238, "Regression_energy": 0.14393487572669983, "Total": 5.296607971191406, "Total_unweighted": 2.6447214568033814}, "train_group": {"cld": {"Classification_binary": 2.7383909526171983, "Classification": 0.01937035994195902, "Regression_pt": 0.1637886006597642, "Regression_eta": 0.016018623121183465, "Regression_sin_phi": 0.008583924537328808, "Regression_cos_phi": 0.008596704989621366, "Regression_energy": 0.1699847796255046, "Total": 6.026458085836233, "Total_unweighted": 3.12473394549256}, "clic": {"Classification_binary": 2.5540310080179913, "Classification": 0.021107318141904776, "Regression_pt": 0.1412660094524572, "Regression_eta": 0.012250393312591725, "Regression_sin_phi": 0.009774447148275411, "Regression_cos_phi": 0.010021500630320947, "Regression_energy": 0.143304987046533, "Total": 5.478794787458317, "Total_unweighted": 2.8917556637500743}, "colliderml": {"Classification_binary": 2.4581506896663354, "Classification": 0.022930280531983118, "Regression_pt": 0.3070600014012139, "Regression_eta": 0.06010460771911137, "Regression_sin_phi": 0.0675881114248578, "Regression_cos_phi": 0.06756825768598565, "Regression_energy": 0.34587179039333676, "Total": 10.038201839000255, "Total_unweighted": 3.3292737388228244}}, "valid": {"Classification_binary": 2.6500940496271306, "Classification": 0.02080898718400435, "Regression_pt": 0.17890285145152698, "Regression_eta": 0.0203932523727417, "Regression_sin_phi": 0.016958600824529473, "Regression_cos_phi": 0.01724949533289129, "Regression_energy": 0.18733002055775036, "Total": 6.451069779829545, "group/cld/Classification": 0.01907598338228591, "group/cld/Classification_binary": 2.7596834872631315, "group/cld/Regression_cos_phi": 0.008713514880931124, "group/cld/Regression_energy": 0.17514415497475483, "group/cld/Regression_eta": 0.015186708024207582, "group/cld/Regression_pt": 0.17014740883035862, "group/cld/Regression_sin_phi": 0.007915860794960185, "group/cld/Total": 6.128454979429853, "group/cld/Total_unweighted": 3.1558671181506295, "group/clic/Classification": 0.021843727598799035, "group/clic/Classification_binary": 2.609348215955369, "group/clic/Regression_cos_phi": 0.009332193973216605, "group/clic/Regression_energy": 0.148437702909429, "group/clic/Regression_eta": 0.012542529309049566, "group/clic/Regression_pt": 0.1446557247892339, "group/clic/Regression_sin_phi": 0.009164355536724659, "group/clic/Total": 5.5970329122340425, "group/clic/Total_unweighted": 2.9553244500718217, "group/colliderml/Classification": 0.02286018244922161, "group/colliderml/Classification_binary": 2.447862148284912, "group/colliderml/Regression_cos_phi": 0.06558100134134293, "group/colliderml/Regression_energy": 0.3373717665672302, "group/colliderml/Regression_eta": 0.058748964220285416, "group/colliderml/Regression_pt": 0.3052229583263397, "group/colliderml/Regression_sin_phi": 0.06641726940870285, "group/colliderml/Total": 9.90748119354248, "group/colliderml/Total_unweighted": 3.304064290598035, "Total_unweighted": 3.091737257350575, "metrics/particle/class_1/efficiency": 0.9460607767105103, "metrics/particle/class_1/pid_accuracy": 0.9823452830314636, "metrics/particle/class_2/efficiency": 0.25888127088546753, "metrics/particle/class_2/pid_accuracy": 0.6596208810806274, "metrics/particle/class_3/efficiency": 0.6287075281143188, "metrics/particle/class_3/pid_accuracy": 0.9718450903892517, "metrics/particle/class_4/efficiency": 0.8954508900642395, "metrics/particle/class_4/pid_accuracy": 0.29411765933036804, "metrics/particle/class_5/efficiency": 0.966810941696167, "metrics/particle/class_5/pid_accuracy": 0.5447761416435242, "metrics/particle/count/bias_mean": -1.7760000228881836, "metrics/particle/count/mae": 5.233142852783203, "metrics/particle/count/prediction_mean": 90.37828826904297, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.06041601672768593, "metrics/particle/event/energy_response_mean": 1.0136891603469849, "metrics/particle/event/met_abs_error": 6.910189151763916, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04999035596847534, "metrics/particle/event/scalar_pt_response_mean": 1.0216504335403442, "metrics/particle/event/vector_pt_closure": 0.05160858854651451, "metrics/particle/matched/delta_eta_abs_mean": 0.01569448783993721, "metrics/particle/matched/delta_phi_abs_mean": 0.0149988466873765, "metrics/particle/matched/delta_r_mean": 0.024274267256259918, "metrics/particle/matched/energy_relative_abs_mean": 0.09626466780900955, "metrics/particle/matched/pid_accuracy": 0.9519757032394409, "metrics/particle/matched/pt_relative_abs_mean": 0.08972811698913574, "metrics/particle/matching/duplicate_fraction": 0.037385717034339905, "metrics/particle/matching/efficiency": 0.7067898511886597, "metrics/particle/matching/f1": 0.7136667370796204, "metrics/particle/matching/purity": 0.7206788063049316, "metrics/particle/matching/target_coverage_dr0p05": 0.6299684047698975, "metrics/particle/matching/target_coverage_dr0p10": 0.7437155246734619, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6288460493087769, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7906492352485657}, "step_train_time": 7.3929760456085205, "step_valid_time": 13.236992292921059, "step_total_time": 17627.618134498596, "elapsed_training_loop_seconds": 17627.618134498596, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.022157409815328, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09683243263701824, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9515477792732167, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9528301886792453, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9521885521885521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04716981132075472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9434724091520862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.944743935309973, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9441077441077441, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9515477792732167, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0164074693713814, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11082010810886112, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1116, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1043, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1038, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9329159212880143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9345878136200717, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9337511190689346, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06541218637992832, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9284436493738819, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9301075268817204, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9292748433303492, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9329159212880143, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0246913095064198, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11963555334784648, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 630, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 552, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 548, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8789808917197452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8761904761904762, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8775834658187599, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12380952380952381, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8726114649681529, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8698412698412699, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8712241653418124, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8789808917197452, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0154751855706845, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06728143365700556, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1506, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1414, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1409, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9477211796246648, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9389110225763613, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9432955303535691, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06108897742363878, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9443699731903485, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9355909694555112, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9399599733155437, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9477211796246648, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0138813409920453, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08627408892861554, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1109, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1038, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1036, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9342934293429342, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9359783588818755, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9351351351351351, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06402164111812443, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9324932493249325, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9341749323715058, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9333333333333333, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9342934293429342, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0171885813245163, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08155268320554987, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 565, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 562, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8883647798742138, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8883647798742138, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8883647798742138, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11163522012578617, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8836477987421384, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8836477987421384, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8836477987421384, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8883647798742138, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9871348744847788, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14605814138170625, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3536, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2650, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2621, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7471102340005639, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7494343891402715, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7482705068473811, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2505656108597285, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.738934310685086, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7412330316742082, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7400818862064097, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7471102340005639, "step_test_time": 34.9407194230007, "step_plot_time": 26.649575822986662, "step_validation_cycle_time": 75.43379685399123} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_150000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_150000.json new file mode 100644 index 0000000000000000000000000000000000000000..19ee6ce2df5e6ca17798b06ed998577159cadcdf --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_150000.json @@ -0,0 +1 @@ +{"epoch": 0.38069616638960446, "train": {"Classification_binary": 2.4628746509552, "Classification": 0.022224990651011467, "Regression_pt": 0.25624528527259827, "Regression_eta": 0.07246304303407669, "Regression_sin_phi": 0.0700029581785202, "Regression_cos_phi": 0.07866355776786804, "Regression_energy": 0.30106812715530396, "Total": 9.473980903625488, "Total_unweighted": 3.263542613014579}, "train_group": {"cld": {"Classification_binary": 2.685629137285479, "Classification": 0.019285367880094873, "Regression_pt": 0.15857773672442538, "Regression_eta": 0.014926900437616327, "Regression_sin_phi": 0.008368110938652142, "Regression_cos_phi": 0.008277415873359766, "Regression_energy": 0.16443826593794264, "Total": 5.872109255633196, "Total_unweighted": 3.0595029350775698}, "clic": {"Classification_binary": 2.5116160968402483, "Classification": 0.02058324469784806, "Regression_pt": 0.13962817485178555, "Regression_eta": 0.012371196622006572, "Regression_sin_phi": 0.00982608064986743, "Regression_cos_phi": 0.009758538969263836, "Regression_energy": 0.14161371292026192, "Total": 5.400319729481374, "Total_unweighted": 2.845397045551282}, "colliderml": {"Classification_binary": 2.430976883379999, "Classification": 0.022683986730338213, "Regression_pt": 0.3008573872154344, "Regression_eta": 0.059797630171308264, "Regression_sin_phi": 0.06715013416941294, "Regression_cos_phi": 0.06730269560378468, "Regression_energy": 0.3400548184346296, "Total": 9.898237165576683, "Total_unweighted": 3.2888235357049074}}, "valid": {"Classification_binary": 2.5813243519176137, "Classification": 0.020444198088212448, "Regression_pt": 0.17467835166237572, "Regression_eta": 0.019849879091436214, "Regression_sin_phi": 0.016814351081848145, "Regression_cos_phi": 0.01714088266546076, "Regression_energy": 0.18274628032337537, "Total": 6.299543346058239, "group/cld/Classification": 0.018823136674596907, "group/cld/Classification_binary": 2.678644058552194, "group/cld/Regression_cos_phi": 0.008656035078332779, "group/cld/Regression_energy": 0.1687195859056838, "group/cld/Regression_eta": 0.014539198672517817, "group/cld/Regression_pt": 0.16394825184598882, "group/cld/Regression_sin_phi": 0.0077704552640306184, "group/cld/Total": 5.9346677090259305, "group/cld/Total_unweighted": 3.061100721993345, "group/clic/Classification": 0.021378177277585293, "group/clic/Classification_binary": 2.538951792615525, "group/clic/Regression_cos_phi": 0.009279298021438275, "group/clic/Regression_energy": 0.14539521805783537, "group/clic/Regression_eta": 0.012102499921271142, "group/clic/Regression_pt": 0.1418999712517921, "group/clic/Regression_sin_phi": 0.009134423859575962, "group/clic/Total": 5.469727211810173, "group/clic/Total_unweighted": 2.8781413810050234, "group/colliderml/Classification": 0.022462474182248116, "group/colliderml/Classification_binary": 2.4199161529541016, "group/colliderml/Regression_cos_phi": 0.06515853106975555, "group/colliderml/Regression_energy": 0.3336685001850128, "group/colliderml/Regression_eta": 0.0582079142332077, "group/colliderml/Regression_pt": 0.3024843633174896, "group/colliderml/Regression_sin_phi": 0.06594054400920868, "group/colliderml/Total": 9.808941841125488, "group/colliderml/Total_unweighted": 3.267838479951024, "Total_unweighted": 3.0129982948303224, "metrics/particle/class_1/efficiency": 0.9525501728057861, "metrics/particle/class_1/pid_accuracy": 0.9808759093284607, "metrics/particle/class_2/efficiency": 0.26430991291999817, "metrics/particle/class_2/pid_accuracy": 0.6574104428291321, "metrics/particle/class_3/efficiency": 0.6319488286972046, "metrics/particle/class_3/pid_accuracy": 0.9703534841537476, "metrics/particle/class_4/efficiency": 0.90343177318573, "metrics/particle/class_4/pid_accuracy": 0.333038866519928, "metrics/particle/class_5/efficiency": 0.9740259647369385, "metrics/particle/class_5/pid_accuracy": 0.5940740704536438, "metrics/particle/count/bias_mean": -0.5274285674095154, "metrics/particle/count/mae": 4.657714366912842, "metrics/particle/count/prediction_mean": 91.6268539428711, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.05357223004102707, "metrics/particle/event/energy_response_mean": 0.9947138428688049, "metrics/particle/event/met_abs_error": 7.327032566070557, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04319828748703003, "metrics/particle/event/scalar_pt_response_mean": 1.0066883563995361, "metrics/particle/event/vector_pt_closure": 0.04926617443561554, "metrics/particle/matched/delta_eta_abs_mean": 0.014443445950746536, "metrics/particle/matched/delta_phi_abs_mean": 0.015172794461250305, "metrics/particle/matched/delta_r_mean": 0.02343141846358776, "metrics/particle/matched/energy_relative_abs_mean": 0.09218835830688477, "metrics/particle/matched/pid_accuracy": 0.9509130716323853, "metrics/particle/matched/pt_relative_abs_mean": 0.087000273168087, "metrics/particle/matching/duplicate_fraction": 0.03786787390708923, "metrics/particle/matching/efficiency": 0.7117008566856384, "metrics/particle/matching/f1": 0.7137433886528015, "metrics/particle/matching/purity": 0.7157976031303406, "metrics/particle/matching/target_coverage_dr0p05": 0.6250573396682739, "metrics/particle/matching/target_coverage_dr0p10": 0.7472933530807495, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6379922032356262, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7980219721794128}, "step_train_time": 7.496488809585571, "step_valid_time": 13.011545847984962, "step_total_time": 18892.513937711716, "elapsed_training_loop_seconds": 18892.513937711716, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0014341792416057, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0864282796786775, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1478, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.949528936742934, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9546684709066305, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9520917678812416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04533152909336942, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9434724091520862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9485791610284168, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9460188933873145, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.949528936742934, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9982002625728066, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09384243386375013, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1114, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1051, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1047, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9400715563506261, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9434470377019749, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9417562724014337, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05655296229802514, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9364937388193202, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9398563734290843, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9381720430107527, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9400715563506261, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058878834814438, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10912829462828944, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 558, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 554, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8885350318471338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8885350318471338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8885350318471338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11146496815286625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8821656050955414, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8821656050955414, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8821656050955414, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8885350318471338, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0005893444194278, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06591834848430889, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1412, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1409, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9463806970509383, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9463806970509383, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9463806970509383, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05361930294906166, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9443699731903485, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9443699731903485, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9443699731903485, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9463806970509383, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994817797476204, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07863132922485705, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1097, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1035, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1035, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9315931593159316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9434822242479489, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9375, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056517775752051046, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9315931593159316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9434822242479489, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9375, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9315931593159316, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004256256825828, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07748611136498273, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 630, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 571, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 566, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8977987421383647, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9063492063492063, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9020537124802528, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09365079365079365, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.889937106918239, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8984126984126984, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8941548183254344, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8977987421383647, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001794550109282, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16268206844741995, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3629, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2678, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2637, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7550042289258528, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.737944337282998, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7463768115942029, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2620556627170019, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7434451649281083, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7266464590796362, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7349498327759197, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7550042289258528, "step_test_time": 35.20287703105714, "step_plot_time": 26.163966630003415, "step_validation_cycle_time": 74.97644502704497} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_160000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_160000.json new file mode 100644 index 0000000000000000000000000000000000000000..04387360ca3b875f6290dc6adb1b84d454abf362 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_160000.json @@ -0,0 +1 @@ +{"epoch": 0.4060759108155781, "train": {"Classification_binary": 2.7361342906951904, "Classification": 0.021607134491205215, "Regression_pt": 0.15204566717147827, "Regression_eta": 0.052550386637449265, "Regression_sin_phi": 0.01308358833193779, "Regression_cos_phi": 0.009581619873642921, "Regression_energy": 0.18039530515670776, "Total": 6.288107872009277, "Total_unweighted": 3.1653979923576117}, "train_group": {"cld": {"Classification_binary": 2.662455096216259, "Classification": 0.018527898956982498, "Regression_pt": 0.15988924735766685, "Regression_eta": 0.016498302384196018, "Regression_sin_phi": 0.008467560508539399, "Regression_cos_phi": 0.008607496980795201, "Regression_energy": 0.16554121095352545, "Total": 5.873197956713374, "Total_unweighted": 3.0399868133579644}, "clic": {"Classification_binary": 2.4849600842049173, "Classification": 0.020350921384475612, "Regression_pt": 0.1385977628218519, "Regression_eta": 0.011665941224441872, "Regression_sin_phi": 0.00983267031503735, "Regression_cos_phi": 0.00973569321538414, "Regression_energy": 0.1400810032859221, "Total": 5.346023233087213, "Total_unweighted": 2.81522407645203}, "colliderml": {"Classification_binary": 2.4107109870280587, "Classification": 0.022370803642559337, "Regression_pt": 0.3003892745073135, "Regression_eta": 0.059448981417102496, "Regression_sin_phi": 0.06707922937544258, "Regression_cos_phi": 0.06673687234431416, "Regression_energy": 0.3395428374722913, "Total": 9.856133924948203, "Total_unweighted": 3.2662789857870824}}, "valid": {"Classification_binary": 2.5518610174005683, "Classification": 0.020110945268110796, "Regression_pt": 0.17183005593039774, "Regression_eta": 0.01968209526755593, "Regression_sin_phi": 0.016889743371443316, "Regression_cos_phi": 0.017201112617145884, "Regression_energy": 0.18003406524658203, "Total": 6.2204351251775565, "group/cld/Classification": 0.018460789893535858, "group/cld/Classification_binary": 2.6428607372527426, "group/cld/Regression_cos_phi": 0.008678697525186742, "group/cld/Regression_energy": 0.16618626168433656, "group/cld/Regression_eta": 0.014417761183799581, "group/cld/Regression_pt": 0.16121303274276408, "group/cld/Regression_sin_phi": 0.007857724707177344, "group/cld/Total": 5.851100191156915, "group/cld/Total_unweighted": 3.019675004989543, "group/clic/Classification": 0.021039138448999285, "group/clic/Classification_binary": 2.50798148297249, "group/clic/Regression_cos_phi": 0.009286293324003828, "group/clic/Regression_energy": 0.14240744773377764, "group/clic/Regression_eta": 0.011823044178333688, "group/clic/Regression_pt": 0.13894612738426695, "group/clic/Regression_sin_phi": 0.009157524463978219, "group/clic/Total": 5.384266630132148, "group/clic/Total_unweighted": 2.8406410585058492, "group/colliderml/Classification": 0.022231705486774445, "group/colliderml/Classification_binary": 2.4134490489959717, "group/colliderml/Regression_cos_phi": 0.06548549234867096, "group/colliderml/Regression_energy": 0.3312405049800873, "group/colliderml/Regression_eta": 0.058232057839632034, "group/colliderml/Regression_pt": 0.29961398243904114, "group/colliderml/Regression_sin_phi": 0.06613468378782272, "group/colliderml/Total": 9.76160717010498, "group/colliderml/Total_unweighted": 3.2563874758780003, "Total_unweighted": 2.977609035101804, "metrics/particle/class_1/efficiency": 0.9487338066101074, "metrics/particle/class_1/pid_accuracy": 0.9814341068267822, "metrics/particle/class_2/efficiency": 0.2769477963447571, "metrics/particle/class_2/pid_accuracy": 0.6713188290596008, "metrics/particle/class_3/efficiency": 0.6400938630104065, "metrics/particle/class_3/pid_accuracy": 0.9700977802276611, "metrics/particle/class_4/efficiency": 0.8643256425857544, "metrics/particle/class_4/pid_accuracy": 0.37211450934410095, "metrics/particle/class_5/efficiency": 0.9696969985961914, "metrics/particle/class_5/pid_accuracy": 0.6339285969734192, "metrics/particle/count/bias_mean": 0.2280000001192093, "metrics/particle/count/mae": 4.4828572273254395, "metrics/particle/count/prediction_mean": 92.38228607177734, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.05198382958769798, "metrics/particle/event/energy_response_mean": 1.017073392868042, "metrics/particle/event/met_abs_error": 6.275477886199951, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046575628221035004, "metrics/particle/event/scalar_pt_response_mean": 1.0298973321914673, "metrics/particle/event/vector_pt_closure": 0.045724764466285706, "metrics/particle/matched/delta_eta_abs_mean": 0.013740976341068745, "metrics/particle/matched/delta_phi_abs_mean": 0.013945754617452621, "metrics/particle/matched/delta_r_mean": 0.021865539252758026, "metrics/particle/matched/energy_relative_abs_mean": 0.09638147801160812, "metrics/particle/matched/pid_accuracy": 0.9520420432090759, "metrics/particle/matched/pt_relative_abs_mean": 0.09009701758623123, "metrics/particle/matching/duplicate_fraction": 0.040453024208545685, "metrics/particle/matching/efficiency": 0.7152663469314575, "metrics/particle/matching/f1": 0.7143825888633728, "metrics/particle/matching/purity": 0.7135010361671448, "metrics/particle/matching/target_coverage_dr0p05": 0.6461151838302612, "metrics/particle/matching/target_coverage_dr0p10": 0.7527872323989868, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.636863648891449, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.800167441368103}, "step_train_time": 7.355810165405273, "step_valid_time": 13.122301486087963, "step_total_time": 20167.8338098526, "elapsed_training_loop_seconds": 20167.8338098526, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0226290355389251, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08628478476310208, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1420, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9629878869448183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9655870445344129, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9642857142857143, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03441295546558704, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.955585464333782, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9581646423751687, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9568733153638814, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9629878869448183, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0229654314753311, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09675785391290914, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1122, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1061, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9490161001788909, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9456327985739751, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9473214285714285, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054367201426024955, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9402852049910874, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9419642857142857, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9490161001788909, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0271016019868529, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10324676867522364, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 559, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 554, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8901273885350318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8929712460063898, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8915470494417863, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10702875399361023, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8821656050955414, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8849840255591054, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8835725677830941, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8901273885350318, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0139423163598826, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06632055301907347, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1490, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1416, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9504021447721179, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9516778523489933, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9510395707578806, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04832214765100671, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9490616621983914, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9503355704697987, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9496981891348089, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9504021447721179, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.014002160128415, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07946362525484563, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1105, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1037, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1035, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9333933393339334, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9384615384615385, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9359205776173285, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06153846153846154, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9315931593159316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9366515837104072, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9341155234657039, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9333933393339334, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0192856018927683, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08503234988943487, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 577, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 573, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9072327044025157, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9072327044025157, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9072327044025157, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09276729559748427, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9009433962264151, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9009433962264151, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9009433962264151, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9072327044025157, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0140970514977727, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1454095868583143, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3656, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2643, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2609, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7451367352692416, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7229212253829321, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7338608912952936, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2770787746170678, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7355511700028193, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7136214442013129, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7244203803970568, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7451367352692416, "step_test_time": 35.30278094706591, "step_plot_time": 26.553029387025163, "step_validation_cycle_time": 75.62451547197998} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_170000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_170000.json new file mode 100644 index 0000000000000000000000000000000000000000..a3595eff5fb62c35b54d22d170842d1b47bafcb8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_170000.json @@ -0,0 +1 @@ +{"epoch": 0.4314556552415517, "train": {"Classification_binary": 2.553913116455078, "Classification": 0.02150203101336956, "Regression_pt": 0.17956086993217468, "Regression_eta": 0.019004996865987778, "Regression_sin_phi": 0.006710503250360489, "Regression_cos_phi": 0.010724928230047226, "Regression_energy": 0.19302062690258026, "Total": 6.196515083312988, "Total_unweighted": 2.984437072649598}, "train_group": {"cld": {"Classification_binary": 2.6292600187811406, "Classification": 0.018412949039122543, "Regression_pt": 0.15460164066370544, "Regression_eta": 0.015675524077322277, "Regression_sin_phi": 0.008432231084849159, "Regression_cos_phi": 0.008449780603719724, "Regression_energy": 0.16106520991425616, "Total": 5.749946542688318, "Total_unweighted": 2.995897354164116}, "clic": {"Classification_binary": 2.4470685363529685, "Classification": 0.02029099164831781, "Regression_pt": 0.13670335701125824, "Regression_eta": 0.011658513645256708, "Regression_sin_phi": 0.009753821156974771, "Regression_cos_phi": 0.009741003364380338, "Regression_energy": 0.1378581632084832, "Total": 5.273188657389429, "Total_unweighted": 2.7730743863876395}, "colliderml": {"Classification_binary": 2.404111212796277, "Classification": 0.02227489969080633, "Regression_pt": 0.2925590863367459, "Regression_eta": 0.05978004281988015, "Regression_sin_phi": 0.06685118155704962, "Regression_cos_phi": 0.0671618254939178, "Regression_energy": 0.33122186429865724, "Total": 9.720923621375281, "Total_unweighted": 3.243960112993334}}, "valid": {"Classification_binary": 2.5275459983132103, "Classification": 0.01975692402232777, "Regression_pt": 0.1709707953713157, "Regression_eta": 0.01998047178441828, "Regression_sin_phi": 0.016748862916773015, "Regression_cos_phi": 0.016864831881089642, "Regression_energy": 0.17874398664994673, "Total": 6.17047119140625, "group/cld/Classification": 0.018128406494221788, "group/cld/Classification_binary": 2.616294373857214, "group/cld/Regression_cos_phi": 0.008464298983837696, "group/cld/Regression_energy": 0.16537522255106174, "group/cld/Regression_eta": 0.014977644098565933, "group/cld/Regression_pt": 0.16119659707901327, "group/cld/Regression_sin_phi": 0.007811609100788198, "group/cld/Total": 5.814765443193152, "group/cld/Total_unweighted": 2.9922481521647026, "group/clic/Classification": 0.02065678226186874, "group/clic/Classification_binary": 2.4857836784200464, "group/clic/Regression_cos_phi": 0.008975268678462251, "group/clic/Regression_energy": 0.14077413843033162, "group/clic/Regression_eta": 0.011929759319792402, "group/clic/Regression_pt": 0.1377936322638329, "group/clic/Regression_sin_phi": 0.009049394663344038, "group/clic/Total": 5.330480372652095, "group/clic/Total_unweighted": 2.8149626540376778, "group/colliderml/Classification": 0.021897342056035995, "group/colliderml/Classification_binary": 2.3895230293273926, "group/colliderml/Regression_cos_phi": 0.06471698731184006, "group/colliderml/Regression_energy": 0.3295513689517975, "group/colliderml/Regression_eta": 0.05832526460289955, "group/colliderml/Regression_pt": 0.29714030027389526, "group/colliderml/Regression_sin_phi": 0.06561923772096634, "group/colliderml/Total": 9.682830810546875, "group/colliderml/Total_unweighted": 3.2267735302448273, "Total_unweighted": 2.950611870939081, "metrics/particle/class_1/efficiency": 0.9456281661987305, "metrics/particle/class_1/pid_accuracy": 0.9815202355384827, "metrics/particle/class_2/efficiency": 0.2718665897846222, "metrics/particle/class_2/pid_accuracy": 0.6583067178726196, "metrics/particle/class_3/efficiency": 0.629741370677948, "metrics/particle/class_3/pid_accuracy": 0.9732224345207214, "metrics/particle/class_4/efficiency": 0.8794892430305481, "metrics/particle/class_4/pid_accuracy": 0.3702359199523926, "metrics/particle/class_5/efficiency": 0.972582995891571, "metrics/particle/class_5/pid_accuracy": 0.574184000492096, "metrics/particle/count/bias_mean": -0.7857142686843872, "metrics/particle/count/mae": 4.657714366912842, "metrics/particle/count/prediction_mean": 91.36856842041016, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.05057257413864136, "metrics/particle/event/energy_response_mean": 0.9896650910377502, "metrics/particle/event/met_abs_error": 6.790400505065918, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04136676341295242, "metrics/particle/event/scalar_pt_response_mean": 1.0049508810043335, "metrics/particle/event/vector_pt_closure": 0.0461687371134758, "metrics/particle/matched/delta_eta_abs_mean": 0.01531357690691948, "metrics/particle/matched/delta_phi_abs_mean": 0.014248990453779697, "metrics/particle/matched/delta_r_mean": 0.023388247936964035, "metrics/particle/matched/energy_relative_abs_mean": 0.09299290180206299, "metrics/particle/matched/pid_accuracy": 0.9522538781166077, "metrics/particle/matched/pt_relative_abs_mean": 0.08772500604391098, "metrics/particle/matching/duplicate_fraction": 0.03591106832027435, "metrics/particle/matching/efficiency": 0.708829939365387, "metrics/particle/matching/f1": 0.7118645906448364, "metrics/particle/matching/purity": 0.7149254083633423, "metrics/particle/matching/target_coverage_dr0p05": 0.6293730735778809, "metrics/particle/matching/target_coverage_dr0p10": 0.7460594177246094, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6347615718841553, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7966081500053406}, "step_train_time": 7.3912353515625, "step_valid_time": 12.474208005005494, "step_total_time": 21446.80611205101, "elapsed_training_loop_seconds": 21446.80611205101, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032658736479294, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0877457184406174, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1422, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9629380053908356, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9622895622895623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03706199460916442, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9569313593539704, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9582210242587601, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9575757575757575, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9616419919246298, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043278414243306, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0930573689397517, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1120, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1050, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9418604651162791, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9401785714285714, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9410187667560321, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.059821428571428574, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9391771019677997, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9375, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.938337801608579, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9418604651162791, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.011128916794416, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0995824048072369, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 559, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 556, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8901273885350318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8958333333333334, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8929712460063898, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10416666666666667, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8853503184713376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8910256410256411, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8881789137380192, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8901273885350318, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0000640232234272, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0625532583593329, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1414, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1412, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9477211796246648, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9528301886792453, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9502688172043011, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04716981132075472, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9463806970509383, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9514824797843666, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9489247311827957, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9477211796246648, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029923425298914, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07984011309938044, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1101, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1035, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1035, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9315931593159316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9400544959128065, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9358047016274864, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05994550408719346, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9315931593159316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9400544959128065, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9358047016274864, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9315931593159316, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005559604375544, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07573673407200499, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 562, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 558, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8836477987421384, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8836477987421384, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8836477987421384, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11635220125786164, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8773584905660378, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8773584905660378, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8773584905660378, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8836477987421384, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9877885924024629, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14456787004799876, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3590, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2701, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2667, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7614885819001973, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7523676880222842, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7569006585400028, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24763231197771587, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.751903016633775, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7428969359331477, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7473728457335015, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7614885819001973, "step_test_time": 35.804526307969354, "step_plot_time": 26.428451432962902, "step_validation_cycle_time": 75.2950574029237} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_180000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_180000.json new file mode 100644 index 0000000000000000000000000000000000000000..b87a5e25108c025a93375fdb4bfb37b0ae6bf68f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_180000.json @@ -0,0 +1 @@ +{"epoch": 0.45683539966752534, "train": {"Classification_binary": 2.412580728530884, "Classification": 0.022171832621097565, "Regression_pt": 0.33580324053764343, "Regression_eta": 0.06528585404157639, "Regression_sin_phi": 0.06765500456094742, "Regression_cos_phi": 0.06364759802818298, "Regression_energy": 0.37777191400527954, "Total": 10.458656311035156, "Total_unweighted": 3.344916172325611}, "train_group": {"cld": {"Classification_binary": 2.5837476017238856, "Classification": 0.01834884450376571, "Regression_pt": 0.15249782142875432, "Regression_eta": 0.014886374109181331, "Regression_sin_phi": 0.007821085631383118, "Regression_cos_phi": 0.007991374042388555, "Regression_energy": 0.15819664160768548, "Total": 5.646525819738348, "Total_unweighted": 2.9434897430470444}, "clic": {"Classification_binary": 2.4023232671233625, "Classification": 0.019560030792971275, "Regression_pt": 0.13014664019908276, "Regression_eta": 0.011230993766320852, "Regression_sin_phi": 0.009085660891232637, "Regression_cos_phi": 0.009437328972224449, "Regression_energy": 0.1313668053608399, "Total": 5.09722001774533, "Total_unweighted": 2.7131507271060347}, "colliderml": {"Classification_binary": 2.391047162923984, "Classification": 0.022057984977915973, "Regression_pt": 0.29462556112669186, "Regression_eta": 0.059012274821093696, "Regression_sin_phi": 0.06616294668418561, "Regression_cos_phi": 0.06614595581359135, "Regression_energy": 0.332994075681635, "Total": 9.711845646361391, "Total_unweighted": 3.2320459620290976}}, "valid": {"Classification_binary": 2.4982682661576705, "Classification": 0.019549614732915704, "Regression_pt": 0.16679911179976029, "Regression_eta": 0.019696617126464845, "Regression_sin_phi": 0.016702739758925004, "Regression_cos_phi": 0.016825225136496803, "Regression_energy": 0.17500749067826704, "Total": 6.070161576704545, "group/cld/Classification": 0.018026283446778642, "group/cld/Classification_binary": 2.5780009817569813, "group/cld/Regression_cos_phi": 0.008351857357836783, "group/cld/Regression_energy": 0.15813950274852998, "group/cld/Regression_eta": 0.014095821279160519, "group/cld/Regression_pt": 0.15335523321273478, "group/cld/Regression_sin_phi": 0.007769230832444861, "group/cld/Total": 5.64350274268617, "group/cld/Total_unweighted": 2.9377389106344673, "group/clic/Classification": 0.02032338558359349, "group/clic/Classification_binary": 2.4581741982317986, "group/clic/Regression_cos_phi": 0.00901806354522705, "group/clic/Regression_energy": 0.14061004557508103, "group/clic/Regression_eta": 0.01227039098739624, "group/clic/Regression_pt": 0.13685353258822827, "group/clic/Regression_sin_phi": 0.00900771643253083, "group/clic/Total": 5.290014713368517, "group/clic/Total_unweighted": 2.786257332943855, "group/colliderml/Classification": 0.02175147645175457, "group/colliderml/Classification_binary": 2.3818345069885254, "group/colliderml/Regression_cos_phi": 0.06464928388595581, "group/colliderml/Regression_energy": 0.3255995512008667, "group/colliderml/Regression_eta": 0.05796346068382263, "group/colliderml/Regression_pt": 0.2942552864551544, "group/colliderml/Regression_sin_phi": 0.06554907560348511, "group/colliderml/Total": 9.615145683288574, "group/colliderml/Total_unweighted": 3.2116026412695646, "Total_unweighted": 2.9128490653905006, "metrics/particle/class_1/efficiency": 0.9568455219268799, "metrics/particle/class_1/pid_accuracy": 0.981058657169342, "metrics/particle/class_2/efficiency": 0.2676539421081543, "metrics/particle/class_2/pid_accuracy": 0.6571474671363831, "metrics/particle/class_3/efficiency": 0.6443690061569214, "metrics/particle/class_3/pid_accuracy": 0.975955069065094, "metrics/particle/class_4/efficiency": 0.9177972674369812, "metrics/particle/class_4/pid_accuracy": 0.34956520795822144, "metrics/particle/class_5/efficiency": 0.9754689931869507, "metrics/particle/class_5/pid_accuracy": 0.6745561957359314, "metrics/particle/count/bias_mean": -1.6222857236862183, "metrics/particle/count/mae": 4.961714267730713, "metrics/particle/count/prediction_mean": 90.53199768066406, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04925244674086571, "metrics/particle/event/energy_response_mean": 0.988501250743866, "metrics/particle/event/met_abs_error": 5.840254306793213, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03932983800768852, "metrics/particle/event/scalar_pt_response_mean": 0.994156539440155, "metrics/particle/event/vector_pt_closure": 0.04295659810304642, "metrics/particle/matched/delta_eta_abs_mean": 0.014931541867554188, "metrics/particle/matched/delta_phi_abs_mean": 0.0140227684751153, "metrics/particle/matched/delta_r_mean": 0.022928833961486816, "metrics/particle/matched/energy_relative_abs_mean": 0.08920693397521973, "metrics/particle/matched/pid_accuracy": 0.9537827968597412, "metrics/particle/matched/pt_relative_abs_mean": 0.08432736992835999, "metrics/particle/matching/duplicate_fraction": 0.035163573920726776, "metrics/particle/matching/efficiency": 0.7195324897766113, "metrics/particle/matching/f1": 0.7259220480918884, "metrics/particle/matching/purity": 0.732426106929779, "metrics/particle/matching/target_coverage_dr0p05": 0.6413034200668335, "metrics/particle/matching/target_coverage_dr0p10": 0.7540274262428284, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6465368866920471, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8011099696159363}, "step_train_time": 13.63933801651001, "step_valid_time": 12.439418929978274, "step_total_time": 22790.977121591568, "elapsed_training_loop_seconds": 22790.977121591568, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007291388732655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07636751487103843, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1407, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9515477792732167, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9592944369063772, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9554054054054054, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0407055630936228, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9468371467025573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9545454545454546, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9506756756756757, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9515477792732167, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9929075871028872, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09015310061282955, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1116, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1056, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1050, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9445438282647585, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.946236559139785, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.945389435989257, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053763440860215055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9391771019677997, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9408602150537635, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9400179051029544, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9445438282647585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999061987546323, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0990728422328564, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 561, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 559, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.893312101910828, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.893312101910828, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.893312101910828, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10668789808917198, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8901273885350318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8901273885350318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8901273885350318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.893312101910828, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9980802848626054, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05773779379112942, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1422, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1417, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.953083109919571, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9524447421299397, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9527638190954774, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047555257870060284, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9497319034852547, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9490957803081045, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9494137353433836, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.953083109919571, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9936916963975828, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07589277833966757, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1104, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1037, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1035, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9333933393339334, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9393115942028986, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9363431151241535, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06068840579710145, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9315931593159316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9375, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9345372460496614, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9333933393339334, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9984487546647465, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07617434770066589, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 634, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 570, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 568, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8962264150943396, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8990536277602523, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8976377952755905, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10094637223974763, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8930817610062893, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8958990536277602, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8944881889763779, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8962264150943396, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9868673319924983, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1424543266113909, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3546, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2672, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2636, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7533126585847195, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7535250987027636, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7534188636684055, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24647490129723632, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7431632365379194, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7433728144388043, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7432680107147892, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7533126585847195, "step_test_time": 35.97294217394665, "step_plot_time": 26.401172359939665, "step_validation_cycle_time": 75.40852975996677} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_190000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_190000.json new file mode 100644 index 0000000000000000000000000000000000000000..f37c21905364ac77de6a3b6809a28107d0910dd5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_190000.json @@ -0,0 +1 @@ +{"epoch": 0.48221514409349897, "train": {"Classification_binary": 2.5493404865264893, "Classification": 0.016305364668369293, "Regression_pt": 0.1369726061820984, "Regression_eta": 0.014657224528491497, "Regression_sin_phi": 0.016266174614429474, "Regression_cos_phi": 0.011302615515887737, "Regression_energy": 0.13016754388809204, "Total": 5.390083312988281, "Total_unweighted": 2.8750120159238577}, "train_group": {"cld": {"Classification_binary": 2.5664487184878593, "Classification": 0.01809695704064951, "Regression_pt": 0.14993306633062706, "Regression_eta": 0.014041377518682958, "Regression_sin_phi": 0.008063029788551617, "Regression_cos_phi": 0.008122982711720832, "Regression_energy": 0.15488892340820706, "Total": 5.578508998105626, "Total_unweighted": 2.9195950552862984}, "clic": {"Classification_binary": 2.3833438211733156, "Classification": 0.019489700995475142, "Regression_pt": 0.12976093702130131, "Regression_eta": 0.011062208488975113, "Regression_sin_phi": 0.00922466017579934, "Regression_cos_phi": 0.009489394650254791, "Regression_energy": 0.13075316992697414, "Total": 5.0708130823599324, "Total_unweighted": 2.6931238924320957}, "colliderml": {"Classification_binary": 2.377197314311076, "Classification": 0.02208163039395222, "Regression_pt": 0.29005711187829486, "Regression_eta": 0.059226465296816896, "Regression_sin_phi": 0.06637126410956139, "Regression_cos_phi": 0.06628189899318211, "Regression_energy": 0.327643951988435, "Total": 9.623785259487393, "Total_unweighted": 3.2088596369713187}}, "valid": {"Classification_binary": 2.4663285688920453, "Classification": 0.019104608622464266, "Regression_pt": 0.16578507856889205, "Regression_eta": 0.019375296072526412, "Regression_sin_phi": 0.01638854525306008, "Regression_cos_phi": 0.016527844559062612, "Regression_energy": 0.17379557869651102, "Total": 6.005016534978693, "group/cld/Classification": 0.01757443331657572, "group/cld/Classification_binary": 2.5366837521816823, "group/cld/Regression_cos_phi": 0.008070365545597482, "group/cld/Regression_energy": 0.15843581138773166, "group/cld/Regression_eta": 0.013873405913089184, "group/cld/Regression_pt": 0.15415245421389315, "group/cld/Regression_sin_phi": 0.007462143263918288, "group/cld/Total": 5.597452366605718, "group/cld/Total_unweighted": 2.8962523658224875, "group/clic/Classification": 0.019771612705068387, "group/clic/Classification_binary": 2.433145076670545, "group/clic/Regression_cos_phi": 0.008795830163549869, "group/clic/Regression_energy": 0.13859083297404837, "group/clic/Regression_eta": 0.011791628725985263, "group/clic/Regression_pt": 0.1347841201944554, "group/clic/Regression_sin_phi": 0.008743458605827169, "group/clic/Total": 5.215239829205452, "group/clic/Total_unweighted": 2.7556225600394795, "group/colliderml/Classification": 0.021640198305249214, "group/colliderml/Classification_binary": 2.357140064239502, "group/colliderml/Regression_cos_phi": 0.0640844851732254, "group/colliderml/Regression_energy": 0.3223288655281067, "group/colliderml/Regression_eta": 0.057814158499240875, "group/colliderml/Regression_pt": 0.2910211384296417, "group/colliderml/Regression_sin_phi": 0.06506727635860443, "group/colliderml/Total": 9.522212982177734, "group/colliderml/Total_unweighted": 3.1790961865335703, "Total_unweighted": 2.8773055206645615, "metrics/particle/class_1/efficiency": 0.9566910266876221, "metrics/particle/class_1/pid_accuracy": 0.9818792343139648, "metrics/particle/class_2/efficiency": 0.27599236369132996, "metrics/particle/class_2/pid_accuracy": 0.676947295665741, "metrics/particle/class_3/efficiency": 0.633625328540802, "metrics/particle/class_3/pid_accuracy": 0.9710493087768555, "metrics/particle/class_4/efficiency": 0.8978451490402222, "metrics/particle/class_4/pid_accuracy": 0.35199999809265137, "metrics/particle/class_5/efficiency": 0.9783549904823303, "metrics/particle/class_5/pid_accuracy": 0.6902654767036438, "metrics/particle/count/bias_mean": -0.645714282989502, "metrics/particle/count/mae": 4.863999843597412, "metrics/particle/count/prediction_mean": 91.5085678100586, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.050381824374198914, "metrics/particle/event/energy_response_mean": 0.9914740324020386, "metrics/particle/event/met_abs_error": 5.960344314575195, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039020199328660965, "metrics/particle/event/scalar_pt_response_mean": 1.0032174587249756, "metrics/particle/event/vector_pt_closure": 0.042691051959991455, "metrics/particle/matched/delta_eta_abs_mean": 0.014936462976038456, "metrics/particle/matched/delta_phi_abs_mean": 0.013753777369856834, "metrics/particle/matched/delta_r_mean": 0.022709595039486885, "metrics/particle/matched/energy_relative_abs_mean": 0.0931461751461029, "metrics/particle/matched/pid_accuracy": 0.952984094619751, "metrics/particle/matched/pt_relative_abs_mean": 0.08702212572097778, "metrics/particle/matching/duplicate_fraction": 0.03413887694478035, "metrics/particle/matching/efficiency": 0.715749979019165, "metrics/particle/matching/f1": 0.7182663679122925, "metrics/particle/matching/purity": 0.7208005785942078, "metrics/particle/matching/target_coverage_dr0p05": 0.6284801959991455, "metrics/particle/matching/target_coverage_dr0p10": 0.7495876550674438, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6430210471153259, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8018292188644409}, "step_train_time": 17.04765796661377, "step_valid_time": 12.572042990010232, "step_total_time": 24114.734157800674, "elapsed_training_loop_seconds": 24114.734157800674, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0123408308049369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07663515306625834, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9609690444145357, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9642133693450371, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9625884732052579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03578663065496286, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9542395693135935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9574611748818366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9558476575665655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9609690444145357, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0085010069385387, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09410010182682538, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1065, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1057, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9525939177101967, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9534467323187108, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9530201342281879, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046553267681289166, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.945438282647585, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.946284691136974, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9458612975391499, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9525939177101967, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0108503524599715, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0825081864849585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 621, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 566, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 565, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9012738853503185, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9114331723027376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9063250600480385, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08856682769726248, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8996815286624203, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9098228663446055, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9047237790232185, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9012738853503185, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0094803161607298, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.058646850122530125, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1419, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1417, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9510723860589813, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9504353650368386, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9507537688442211, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04956463496316142, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9497319034852547, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9490957803081045, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9494137353433836, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9510723860589813, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0083077852535114, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.072991288077207, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1104, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1042, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1039, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9378937893789379, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9438405797101449, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9408577878103838, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05615942028985507, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9351935193519352, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9411231884057971, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9381489841986456, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9378937893789379, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0083625005692611, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07100199055400935, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 639, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 579, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 577, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.910377358490566, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9061032863849765, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.908235294117647, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09389671361502347, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9072327044025157, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9029733959311425, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9050980392156863, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.910377358490566, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9896677527435408, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13909947910729503, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3588, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2674, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2633, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7538765153650973, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7452619843924192, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7495444989488437, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2547380156075808, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7423174513673527, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7338350055741361, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.738051857042747, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7538765153650973, "step_test_time": 35.43805075797718, "step_plot_time": 27.15221913799178, "step_validation_cycle_time": 75.76058281806763} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..374424c9e615b64690f860584f3222328b1c3c36 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.05075948885194726, "train": {"Classification_binary": 3.870072841644287, "Classification": 0.03104422427713871, "Regression_pt": 0.22454941272735596, "Regression_eta": 0.016973789781332016, "Regression_sin_phi": 0.014718259684741497, "Regression_cos_phi": 0.013666770420968533, "Regression_energy": 0.23274950683116913, "Total": 8.465923309326172, "Total_unweighted": 4.403774805366993}, "train_group": {"cld": {"Classification_binary": 4.069721844819215, "Classification": 0.031173453730595362, "Regression_pt": 0.26954835008930517, "Regression_eta": 0.027523896243009303, "Regression_sin_phi": 0.014035861014861154, "Regression_cos_phi": 0.014104346249688853, "Regression_energy": 0.27834341207424085, "Total": 9.46745254327585, "Total_unweighted": 4.704451164220916}, "clic": {"Classification_binary": 3.7766253284351556, "Classification": 0.03387688197931665, "Regression_pt": 0.22182303546609994, "Regression_eta": 0.021676517710714282, "Regression_sin_phi": 0.016393737092301873, "Regression_cos_phi": 0.0163957489777111, "Regression_energy": 0.22421222367507967, "Total": 8.408860437884302, "Total_unweighted": 4.311003473336379}, "colliderml": {"Classification_binary": 3.111351282388956, "Classification": 0.031522478483192194, "Regression_pt": 0.4314373185684731, "Regression_eta": 0.06911014285576236, "Regression_sin_phi": 0.0792859884666967, "Regression_cos_phi": 0.07984606333233572, "Regression_energy": 0.48218697739077043, "Total": 13.265927841713479, "Total_unweighted": 4.284740251486187}}, "valid": {"Classification_binary": 3.7802515203302556, "Classification": 0.03293348008936102, "Regression_pt": 0.274218073758212, "Regression_eta": 0.030365599285472524, "Regression_sin_phi": 0.023175935311750932, "Regression_cos_phi": 0.02377295060591264, "Regression_energy": 0.2860572641546076, "Total": 9.541831276633523, "group/cld/Classification": 0.031626396990836934, "group/cld/Classification_binary": 4.047320102123504, "group/cld/Regression_cos_phi": 0.014447243923836567, "group/cld/Regression_energy": 0.286557359898344, "group/cld/Regression_eta": 0.026291961365557732, "group/cld/Regression_pt": 0.2767618666303919, "group/cld/Regression_sin_phi": 0.012654328599889228, "group/cld/Total": 9.555803014876995, "group/cld/Total_unweighted": 4.69565925953236, "group/clic/Classification": 0.03482188316101723, "group/clic/Classification_binary": 3.7521943437292222, "group/clic/Regression_cos_phi": 0.015241175255876906, "group/clic/Regression_energy": 0.22689504826322515, "group/clic/Regression_eta": 0.022048592567443848, "group/clic/Regression_pt": 0.2226435275788003, "group/clic/Regression_sin_phi": 0.015378822671606186, "group/clic/Total": 8.39882643679355, "group/clic/Total_unweighted": 4.289223393227192, "group/colliderml/Classification": 0.03122587315738201, "group/colliderml/Classification_binary": 3.0781617164611816, "group/colliderml/Regression_cos_phi": 0.0762292817234993, "group/colliderml/Regression_energy": 0.45837706327438354, "group/colliderml/Regression_eta": 0.06676309555768967, "group/colliderml/Regression_pt": 0.41824570298194885, "group/colliderml/Regression_sin_phi": 0.07698717713356018, "group/colliderml/Total": 12.858365058898926, "group/colliderml/Total_unweighted": 4.205989910289645, "Total_unweighted": 4.450774823535572, "metrics/particle/class_1/efficiency": 0.9212465882301331, "metrics/particle/class_1/pid_accuracy": 0.9738192558288574, "metrics/particle/class_2/efficiency": 0.14540085196495056, "metrics/particle/class_2/pid_accuracy": 0.22102747857570648, "metrics/particle/class_3/efficiency": 0.5051063895225525, "metrics/particle/class_3/pid_accuracy": 0.9575150609016418, "metrics/particle/class_4/efficiency": 0.8747007250785828, "metrics/particle/class_4/pid_accuracy": 0.005474452394992113, "metrics/particle/class_5/efficiency": 0.9711399674415588, "metrics/particle/class_5/pid_accuracy": 0.06389301270246506, "metrics/particle/count/bias_mean": -0.8925714492797852, "metrics/particle/count/mae": 6.905142784118652, "metrics/particle/count/prediction_mean": 91.26171112060547, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.10694712400436401, "metrics/particle/event/energy_response_mean": 1.0307785272598267, "metrics/particle/event/met_abs_error": 16.677547454833984, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09619441628456116, "metrics/particle/event/scalar_pt_response_mean": 1.0327192544937134, "metrics/particle/event/vector_pt_closure": 0.1045648604631424, "metrics/particle/matched/delta_eta_abs_mean": 0.02749798446893692, "metrics/particle/matched/delta_phi_abs_mean": 0.020137052983045578, "metrics/particle/matched/delta_r_mean": 0.03778141364455223, "metrics/particle/matched/energy_relative_abs_mean": 0.10800246894359589, "metrics/particle/matched/pid_accuracy": 0.9264086484909058, "metrics/particle/matched/pt_relative_abs_mean": 0.10111002624034882, "metrics/particle/matching/duplicate_fraction": 0.04720490053296089, "metrics/particle/matching/efficiency": 0.6256278157234192, "metrics/particle/matching/f1": 0.6286723613739014, "metrics/particle/matching/purity": 0.6317467093467712, "metrics/particle/matching/target_coverage_dr0p05": 0.4698393940925598, "metrics/particle/matching/target_coverage_dr0p10": 0.6637130379676819, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5703664422035217, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7387300729751587}, "step_train_time": 7.2956531047821045, "step_valid_time": 12.815511861001141, "step_total_time": 2447.510182619095, "elapsed_training_loop_seconds": 2447.510182619095, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0157587438807618, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2233642401228435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1506, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1337, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1240, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8997308209959624, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8877822045152722, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8937165775401069, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11221779548472775, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8344549125168237, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8233731739707836, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8288770053475936, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8997308209959624, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0230394137799608, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22474288737457337, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1112, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 925, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 885, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8273703041144902, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8318345323741008, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8295964125560538, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1681654676258993, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7915921288014311, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.795863309352518, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7937219730941704, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8273703041144902, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.027537031092959, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2094927666384543, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 632, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 487, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 463, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7754777070063694, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7705696202531646, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.773015873015873, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.22943037974683544, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7372611464968153, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7325949367088608, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.734920634920635, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7754777070063694, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0117918773215344, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17743945475540457, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1510, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1348, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1265, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.903485254691689, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8927152317880794, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8980679546968687, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10728476821192053, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8478552278820375, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8377483443708609, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8427714856762158, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.903485254691689, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0090172369688035, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17462122333826313, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1090, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 933, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 902, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8397839783978398, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8559633027522936, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8477964561562926, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.14403669724770643, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8118811881188119, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8275229357798165, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8196274420717855, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8397839783978398, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0111571852371466, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18838157474596218, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 648, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 499, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 475, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7845911949685535, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7700617283950617, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7772585669781932, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.22993827160493827, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7468553459119497, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7330246913580247, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7398753894080997, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7845911949685535, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.029234251169118, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3234311163844048, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3608, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2452, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2246, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.6912884127431632, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.6796008869179601, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.6853948287910552, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.3203991130820399, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6332111643642515, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6225055432372506, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6278127183787561, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.6912884127431632, "step_test_time": 33.35776425595395, "step_plot_time": 26.3946718609659, "step_validation_cycle_time": 73.10210944793653} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_200000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_200000.json new file mode 100644 index 0000000000000000000000000000000000000000..bce0ef1d8d48e7ce5e10465514b1babf9b38b0db --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_200000.json @@ -0,0 +1 @@ +{"epoch": 0.5075948885194727, "train": {"Classification_binary": 2.404949426651001, "Classification": 0.023904109373688698, "Regression_pt": 0.22850288450717926, "Regression_eta": 0.011873586103320122, "Regression_sin_phi": 0.011455659754574299, "Regression_cos_phi": 0.0071798283606767654, "Regression_energy": 0.215748131275177, "Total": 6.685286521911621, "Total_unweighted": 2.903613626025617}, "train_group": {"cld": {"Classification_binary": 2.5235686094553262, "Classification": 0.01732073325451251, "Regression_pt": 0.14879176629198207, "Regression_eta": 0.01399954313142421, "Regression_sin_phi": 0.007808790199581657, "Regression_cos_phi": 0.008083286996533698, "Regression_energy": 0.1530586182399913, "Total": 5.498096678708051, "Total_unweighted": 2.8726313475693517}, "clic": {"Classification_binary": 2.3687515712069893, "Classification": 0.01922274083122165, "Regression_pt": 0.12879863269850164, "Regression_eta": 0.010673798304915562, "Regression_sin_phi": 0.009144270225863034, "Regression_cos_phi": 0.009166683499076721, "Regression_energy": 0.12973226015706976, "Total": 5.02935347585621, "Total_unweighted": 2.675489956923638}, "colliderml": {"Classification_binary": 2.3642636627048343, "Classification": 0.021776780637132155, "Regression_pt": 0.2875960066064342, "Regression_eta": 0.05925427669206181, "Regression_sin_phi": 0.06595278282960255, "Regression_cos_phi": 0.0660313962995111, "Regression_energy": 0.3250928498782195, "Total": 9.55786573564684, "Total_unweighted": 3.1899677556477957}}, "valid": {"Classification_binary": 2.4474165482954544, "Classification": 0.018985403667796743, "Regression_pt": 0.16386443051424893, "Regression_eta": 0.019858486002141778, "Regression_sin_phi": 0.01633404710076072, "Regression_cos_phi": 0.01648044152693315, "Regression_energy": 0.17232778722589667, "Total": 5.957579456676136, "group/cld/Classification": 0.017307910513370595, "group/cld/Classification_binary": 2.5105185001454453, "group/cld/Regression_cos_phi": 0.007947350436068595, "group/cld/Regression_energy": 0.15670773323546064, "group/cld/Regression_eta": 0.014335344446466324, "group/cld/Regression_pt": 0.15197682887949843, "group/cld/Regression_sin_phi": 0.007406934778741065, "group/cld/Total": 5.5355237595578455, "group/cld/Total_unweighted": 2.8662006024350513, "group/clic/Classification": 0.019796573101206028, "group/clic/Classification_binary": 2.4168420345225234, "group/clic/Regression_cos_phi": 0.008761051487415395, "group/clic/Regression_energy": 0.1380390207818214, "group/clic/Regression_eta": 0.012338620551089023, "group/clic/Regression_pt": 0.13378068234058135, "group/clic/Regression_sin_phi": 0.00863890825433934, "group/clic/Total": 5.187294493330286, "group/clic/Total_unweighted": 2.7381968910389753, "group/colliderml/Classification": 0.021530242636799812, "group/colliderml/Classification_binary": 2.35186767578125, "group/colliderml/Regression_cos_phi": 0.06422208249568939, "group/colliderml/Regression_energy": 0.31893491744995117, "group/colliderml/Regression_eta": 0.05817233398556709, "group/colliderml/Regression_pt": 0.2871554493904114, "group/colliderml/Regression_sin_phi": 0.06516192108392715, "group/colliderml/Total": 9.4600830078125, "group/colliderml/Total_unweighted": 3.167044622823596, "Total_unweighted": 2.8552671443332325, "metrics/particle/class_1/efficiency": 0.9529673457145691, "metrics/particle/class_1/pid_accuracy": 0.9818246960639954, "metrics/particle/class_2/efficiency": 0.2763832211494446, "metrics/particle/class_2/pid_accuracy": 0.6742614507675171, "metrics/particle/class_3/efficiency": 0.6348268389701843, "metrics/particle/class_3/pid_accuracy": 0.9741631746292114, "metrics/particle/class_4/efficiency": 0.889066219329834, "metrics/particle/class_4/pid_accuracy": 0.43177738785743713, "metrics/particle/class_5/efficiency": 0.9754689931869507, "metrics/particle/class_5/pid_accuracy": 0.665680468082428, "metrics/particle/count/bias_mean": -2.418285608291626, "metrics/particle/count/mae": 5.005714416503906, "metrics/particle/count/prediction_mean": 89.73600006103516, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04990934208035469, "metrics/particle/event/energy_response_mean": 0.9966202974319458, "metrics/particle/event/met_abs_error": 6.34729528427124, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04032657667994499, "metrics/particle/event/scalar_pt_response_mean": 1.0157992839813232, "metrics/particle/event/vector_pt_closure": 0.04356805980205536, "metrics/particle/matched/delta_eta_abs_mean": 0.016448304057121277, "metrics/particle/matched/delta_phi_abs_mean": 0.014209662564098835, "metrics/particle/matched/delta_r_mean": 0.024216551333665848, "metrics/particle/matched/energy_relative_abs_mean": 0.0907447561621666, "metrics/particle/matched/pid_accuracy": 0.9546542763710022, "metrics/particle/matched/pt_relative_abs_mean": 0.08691562712192535, "metrics/particle/matching/duplicate_fraction": 0.03643067181110382, "metrics/particle/matching/efficiency": 0.7147640585899353, "metrics/particle/matching/f1": 0.7242670655250549, "metrics/particle/matching/purity": 0.7340261340141296, "metrics/particle/matching/target_coverage_dr0p05": 0.6298877596855164, "metrics/particle/matching/target_coverage_dr0p10": 0.7493768334388733, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.639790415763855, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7976065278053284}, "step_train_time": 14.840303897857666, "step_valid_time": 12.616860781097785, "step_total_time": 25453.283433437347, "elapsed_training_loop_seconds": 25453.283433437347, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.016881082784743, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08650328829320142, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1477, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9535666218034994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.95937711577522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9564630442119474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.040622884224779957, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.949528936742934, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9553148273527421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9524130948363145, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9535666218034994, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0130306542347909, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0953931495138185, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1122, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1057, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.945438282647585, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9420677361853832, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.94375, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.057932263814616754, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9418604651162791, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9385026737967914, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9401785714285714, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.945438282647585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0159073102656104, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09074843452395487, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 567, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 564, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9028662420382165, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9014308426073132, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9021479713603818, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0985691573926868, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8980891719745223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8966613672496025, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8973747016706444, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9028662420382165, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.011870170143044, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.059230699282316834, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1489, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1393, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9376675603217158, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9395567494963063, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9386112042938611, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06044325050369376, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9336461126005362, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9355271994627267, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9345857094934585, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9376675603217158, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.010429387380234, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07918752391708983, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1104, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1036, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1034, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9324932493249325, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9384057971014492, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9354401805869075, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06159420289855073, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9306930693069307, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9365942028985508, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9336343115124154, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9324932493249325, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0142816152620042, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07553252577475278, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 637, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 577, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 574, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9072327044025157, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9058084772370487, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9065200314218381, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09419152276295134, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9025157232704403, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9010989010989011, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9018067556952082, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9072327044025157, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002025667744907, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13304637079451376, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3546, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2658, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2632, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.749365661122075, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7495769881556683, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7494713097419992, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.25042301184433163, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7420355229771638, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7422447828539199, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7421401381643874, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.749365661122075, "step_test_time": 35.5466993690934, "step_plot_time": 26.777209975058213, "step_validation_cycle_time": 75.52990776591469} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_210000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_210000.json new file mode 100644 index 0000000000000000000000000000000000000000..0d3b2ed8328066a8fd759d515b13f6acd439ac9c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_210000.json @@ -0,0 +1 @@ +{"epoch": 0.5329746329454462, "train": {"Classification_binary": 2.3789730072021484, "Classification": 0.022950630635023117, "Regression_pt": 0.2667296826839447, "Regression_eta": 0.05671795830130577, "Regression_sin_phi": 0.06404925882816315, "Regression_cos_phi": 0.06384318321943283, "Regression_energy": 0.29364290833473206, "Total": 9.113268852233887, "Total_unweighted": 3.14690662920475}, "train_group": {"cld": {"Classification_binary": 2.510969028458581, "Classification": 0.017418170230345684, "Regression_pt": 0.14998265411760714, "Regression_eta": 0.015372438503460059, "Regression_sin_phi": 0.007882572981101033, "Regression_cos_phi": 0.008093117736279964, "Regression_energy": 0.154921445760641, "Total": 5.519803934985095, "Total_unweighted": 2.864639427788016}, "clic": {"Classification_binary": 2.358007171132543, "Classification": 0.019019513997856202, "Regression_pt": 0.12878848153639127, "Regression_eta": 0.011098730330307921, "Regression_sin_phi": 0.009227253793103917, "Regression_cos_phi": 0.009382661921461602, "Regression_energy": 0.12989351934230364, "Total": 5.0228924930274665, "Total_unweighted": 2.6654173320539676}, "colliderml": {"Classification_binary": 2.355722698622835, "Classification": 0.021839557845747756, "Regression_pt": 0.2894163038470074, "Regression_eta": 0.05957617667576153, "Regression_sin_phi": 0.06632516159491982, "Regression_cos_phi": 0.06623476256287383, "Regression_energy": 0.3281185241070336, "Total": 9.596736650980874, "Total_unweighted": 3.187233185256179}}, "valid": {"Classification_binary": 2.4214197332208807, "Classification": 0.01876136389645663, "Regression_pt": 0.16281779896129261, "Regression_eta": 0.018934297561645507, "Regression_sin_phi": 0.01624383276159113, "Regression_cos_phi": 0.016368845376101407, "Regression_energy": 0.17032305977561257, "Total": 5.897291148792614, "group/cld/Classification": 0.017226877364706485, "group/cld/Classification_binary": 2.481368693899601, "group/cld/Regression_cos_phi": 0.007929481724475293, "group/cld/Regression_energy": 0.1563730645687022, "group/cld/Regression_eta": 0.013327915617760191, "group/cld/Regression_pt": 0.15217344811622133, "group/cld/Regression_sin_phi": 0.007290341118548779, "group/cld/Total": 5.497469638256317, "group/cld/Total_unweighted": 2.8356898224100155, "group/clic/Classification": 0.019432403939835568, "group/clic/Classification_binary": 2.389287583371426, "group/clic/Regression_cos_phi": 0.00861459716837457, "group/clic/Regression_energy": 0.13464091686492272, "group/clic/Regression_eta": 0.011398990103539, "group/clic/Regression_pt": 0.13182548766440533, "group/clic/Regression_sin_phi": 0.008654007252226485, "group/clic/Total": 5.106287205472905, "group/clic/Total_unweighted": 2.7038539863647295, "group/colliderml/Classification": 0.021297717466950417, "group/colliderml/Classification_binary": 2.3397152423858643, "group/colliderml/Regression_cos_phi": 0.06393761187791824, "group/colliderml/Regression_energy": 0.316118061542511, "group/colliderml/Regression_eta": 0.05753800645470619, "group/colliderml/Regression_pt": 0.28512561321258545, "group/colliderml/Regression_sin_phi": 0.06483980268239975, "group/colliderml/Total": 9.395330429077148, "group/colliderml/Total_unweighted": 3.1485720556229353, "Total_unweighted": 2.8248689315535804, "metrics/particle/class_1/efficiency": 0.9571545720100403, "metrics/particle/class_1/pid_accuracy": 0.9812423586845398, "metrics/particle/class_2/efficiency": 0.2819421589374542, "metrics/particle/class_2/pid_accuracy": 0.6903881430625916, "metrics/particle/class_3/efficiency": 0.6506699323654175, "metrics/particle/class_3/pid_accuracy": 0.9756511449813843, "metrics/particle/class_4/efficiency": 0.8938547372817993, "metrics/particle/class_4/pid_accuracy": 0.3991071283817291, "metrics/particle/class_5/efficiency": 0.9711399674415588, "metrics/particle/class_5/pid_accuracy": 0.7087666988372803, "metrics/particle/count/bias_mean": -1.2285714149475098, "metrics/particle/count/mae": 4.817142963409424, "metrics/particle/count/prediction_mean": 90.92571258544922, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.048380374908447266, "metrics/particle/event/energy_response_mean": 0.9990423321723938, "metrics/particle/event/met_abs_error": 5.902508735656738, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03897513076663017, "metrics/particle/event/scalar_pt_response_mean": 1.0042614936828613, "metrics/particle/event/vector_pt_closure": 0.0435648113489151, "metrics/particle/matched/delta_eta_abs_mean": 0.01322114560753107, "metrics/particle/matched/delta_phi_abs_mean": 0.01400199718773365, "metrics/particle/matched/delta_r_mean": 0.021491460502147675, "metrics/particle/matched/energy_relative_abs_mean": 0.0907156765460968, "metrics/particle/matched/pid_accuracy": 0.9556957483291626, "metrics/particle/matched/pt_relative_abs_mean": 0.08550851792097092, "metrics/particle/matching/duplicate_fraction": 0.0353192575275898, "metrics/particle/matching/efficiency": 0.7242884635925293, "metrics/particle/matching/f1": 0.7291488647460938, "metrics/particle/matching/purity": 0.7340748906135559, "metrics/particle/matching/target_coverage_dr0p05": 0.6562224626541138, "metrics/particle/matching/target_coverage_dr0p10": 0.7585849761962891, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6483598947525024, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8058101534843445}, "step_train_time": 12.508558511734009, "step_valid_time": 12.598942846991122, "step_total_time": 26799.474455833435, "elapsed_training_loop_seconds": 26799.474455833435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0067042408213607, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.080114034927945, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9576043068640646, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9576043068640646, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9576043068640646, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0423956931359354, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9535666218034994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9535666218034994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9535666218034994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9576043068640646, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042209810675893, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09312814179226259, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1061, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9490161001788909, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9498657117278424, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9494407158836689, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050134288272157566, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9444941808415398, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9440715883668904, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9490161001788909, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0098463347264017, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0926637666699257, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 569, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 568, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9060509554140127, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9074960127591707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9067729083665339, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09250398724082935, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9044585987261147, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9059011164274322, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9051792828685259, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9060509554140127, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0001825373005446, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06122308975056323, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1494, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1424, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1421, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9544235924932976, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9531459170013387, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9537843268586738, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04685408299866131, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9524128686327078, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9511378848728246, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9517749497655726, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9544235924932976, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9977153141290562, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07487179098617636, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1105, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1042, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1039, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9378937893789379, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9429864253393665, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9404332129963899, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05701357466063348, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9351935193519352, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9402714932126697, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9377256317689531, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9378937893789379, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0016182556983628, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07761277300844793, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 634, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 579, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 575, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.910377358490566, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9132492113564669, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9118110236220472, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08675078864353312, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9040880503144654, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9069400630914827, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.905511811023622, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.910377358490566, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9948140158382678, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.133424663319212, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3562, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2683, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2648, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7564138708767973, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.753228523301516, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7548178365452244, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.246771476698484, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.746546377220186, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7434025828186412, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7449711633141088, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7564138708767973, "step_test_time": 35.5953567790566, "step_plot_time": 27.528300347039476, "step_validation_cycle_time": 76.32613629789557} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_220000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_220000.json new file mode 100644 index 0000000000000000000000000000000000000000..af117251df3f00d6961bf9f834fcc071a1c77a81 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_220000.json @@ -0,0 +1 @@ +{"epoch": 0.5583543773714199, "train": {"Classification_binary": 2.793950080871582, "Classification": 0.01619591750204563, "Regression_pt": 0.17148329317569733, "Regression_eta": 0.008147897198796272, "Regression_sin_phi": 0.00670118210837245, "Regression_cos_phi": 0.00684712827205658, "Regression_energy": 0.1727636158466339, "Total": 6.042962074279785, "Total_unweighted": 3.176089114975184}, "train_group": {"cld": {"Classification_binary": 2.4728490285530778, "Classification": 0.0175430229725595, "Regression_pt": 0.14656235955789418, "Regression_eta": 0.01622215023927926, "Regression_sin_phi": 0.00792207776061343, "Regression_cos_phi": 0.008171183499089348, "Regression_energy": 0.15214028729203932, "Total": 5.4378992697435935, "Total_unweighted": 2.8214101098745528}, "clic": {"Classification_binary": 2.347149757651595, "Classification": 0.018610506089174892, "Regression_pt": 0.12720504705164884, "Regression_eta": 0.010919089154298376, "Regression_sin_phi": 0.009162326618838866, "Regression_cos_phi": 0.009148049722048076, "Regression_energy": 0.12848068357587936, "Total": 4.977025336331433, "Total_unweighted": 2.650675459863484}, "colliderml": {"Classification_binary": 2.348647691824057, "Classification": 0.021613859814835025, "Regression_pt": 0.2877761347068323, "Regression_eta": 0.05891049539183711, "Regression_sin_phi": 0.06595650179921328, "Regression_cos_phi": 0.06572677713032957, "Regression_energy": 0.3256124830818749, "Total": 9.539615227295473, "Total_unweighted": 3.1742439437489787}}, "valid": {"Classification_binary": 2.403304776278409, "Classification": 0.018651372736150567, "Regression_pt": 0.16019550670276989, "Regression_eta": 0.018915486335754395, "Regression_sin_phi": 0.01613582264293324, "Regression_cos_phi": 0.016373800147663462, "Regression_energy": 0.16796635714444247, "Total": 5.83507080078125, "group/cld/Classification": 0.017055013078324337, "group/cld/Classification_binary": 2.4562395785717253, "group/cld/Regression_cos_phi": 0.007930261023501133, "group/cld/Regression_energy": 0.1494693654648801, "group/cld/Regression_eta": 0.01359482394888046, "group/cld/Regression_pt": 0.1449150227485819, "group/cld/Regression_sin_phi": 0.007344201524206933, "group/cld/Total": 5.35562458444149, "group/cld/Total_unweighted": 2.7965482663601, "group/clic/Classification": 0.019352595856849182, "group/clic/Classification_binary": 2.36934694330743, "group/clic/Regression_cos_phi": 0.008707083920215039, "group/clic/Regression_energy": 0.1363624613335792, "group/clic/Regression_eta": 0.011165736837589995, "group/clic/Regression_pt": 0.13324666530527968, "group/clic/Regression_sin_phi": 0.008558722886633365, "group/clic/Total": 5.109073720079787, "group/clic/Total_unweighted": 2.686740209447577, "group/colliderml/Classification": 0.021280795335769653, "group/colliderml/Classification_binary": 2.3475608825683594, "group/colliderml/Regression_cos_phi": 0.06369767338037491, "group/colliderml/Regression_energy": 0.31513795256614685, "group/colliderml/Regression_eta": 0.05730987712740898, "group/colliderml/Regression_pt": 0.28424403071403503, "group/colliderml/Regression_sin_phi": 0.06421895325183868, "group/colliderml/Total": 9.376054763793945, "group/colliderml/Total_unweighted": 3.1534501649439335, "Total_unweighted": 2.801543121988123, "metrics/particle/class_1/efficiency": 0.9595031142234802, "metrics/particle/class_1/pid_accuracy": 0.9817874431610107, "metrics/particle/class_2/efficiency": 0.2925388813018799, "metrics/particle/class_2/pid_accuracy": 0.6719121336936951, "metrics/particle/class_3/efficiency": 0.6579347848892212, "metrics/particle/class_3/pid_accuracy": 0.9758138060569763, "metrics/particle/class_4/efficiency": 0.9010375142097473, "metrics/particle/class_4/pid_accuracy": 0.41984057426452637, "metrics/particle/class_5/efficiency": 0.9769119620323181, "metrics/particle/class_5/pid_accuracy": 0.6336780190467834, "metrics/particle/count/bias_mean": 1.2605714797973633, "metrics/particle/count/mae": 4.396571636199951, "metrics/particle/count/prediction_mean": 93.41485595703125, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04888715595006943, "metrics/particle/event/energy_response_mean": 1.0112831592559814, "metrics/particle/event/met_abs_error": 5.725227355957031, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0392327718436718, "metrics/particle/event/scalar_pt_response_mean": 1.0150829553604126, "metrics/particle/event/vector_pt_closure": 0.0411657989025116, "metrics/particle/matched/delta_eta_abs_mean": 0.01298604253679514, "metrics/particle/matched/delta_phi_abs_mean": 0.01419700589030981, "metrics/particle/matched/delta_r_mean": 0.021486585959792137, "metrics/particle/matched/energy_relative_abs_mean": 0.089970663189888, "metrics/particle/matched/pid_accuracy": 0.9542786478996277, "metrics/particle/matched/pt_relative_abs_mean": 0.0850389301776886, "metrics/particle/matching/duplicate_fraction": 0.03714306652545929, "metrics/particle/matching/efficiency": 0.7300490140914917, "metrics/particle/matching/f1": 0.7250897884368896, "metrics/particle/matching/purity": 0.7201974391937256, "metrics/particle/matching/target_coverage_dr0p05": 0.6643021106719971, "metrics/particle/matching/target_coverage_dr0p10": 0.7660444974899292, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6540459990501404, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.811893105506897}, "step_train_time": 17.31827139854431, "step_valid_time": 12.62452607601881, "step_total_time": 28134.22004389763, "elapsed_training_loop_seconds": 28134.22004389763, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0093335087271167, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07582186850934158, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1408, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9542395693135935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9568151147098516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9555256064690026, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043184885290148446, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9475100942126514, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9500674763832658, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9487870619946092, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9542395693135935, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0064251814839258, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09436578400424112, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1121, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1062, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9499105545617174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9473684210526315, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9486377847253238, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05263157894736842, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9411239964317574, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9423849933005806, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9499105545617174, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0099678395431386, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0859317631944867, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 574, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 573, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9140127388535032, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9169329073482428, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9154704944178629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08306709265175719, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9124203821656051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9153354632587859, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9138755980861244, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9140127388535032, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0050680341450515, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06239547459359196, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1490, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1421, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1419, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9524128686327078, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9536912751677852, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9530516431924883, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046308724832214765, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9510723860589813, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9523489932885906, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9517102615694165, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9524128686327078, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023975151500268, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.075974053180345, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1105, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1039, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1037, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9351935193519352, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9402714932126697, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9377256317689531, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05972850678733032, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9333933393339334, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9384615384615385, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9359205776173285, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9351935193519352, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0057950762638772, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08005698488605206, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 581, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 579, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9135220125786163, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9135220125786163, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9135220125786163, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08647798742138364, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.910377358490566, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.910377358490566, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.910377358490566, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9135220125786163, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003158447747167, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13900316416913594, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3666, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2673, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2633, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7535945869749083, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7291325695581015, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7411617912103147, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.27086743044189854, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7423174513673527, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7182214948172395, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7300707056703175, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7535945869749083, "step_test_time": 35.689437986933626, "step_plot_time": 27.231358525925316, "step_validation_cycle_time": 76.12354167900048} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_230000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_230000.json new file mode 100644 index 0000000000000000000000000000000000000000..7affd0fa84fef084663252a3beb3a0a2d0611e3a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_230000.json @@ -0,0 +1 @@ +{"epoch": 0.5837341217973935, "train": {"Classification_binary": 2.1950767040252686, "Classification": 0.015979157760739326, "Regression_pt": 0.10582108795642853, "Regression_eta": 0.011830970644950867, "Regression_sin_phi": 0.010745899751782417, "Regression_cos_phi": 0.009692173451185226, "Regression_energy": 0.10243631154298782, "Total": 4.439788818359375, "Total_unweighted": 2.4515823051333427}, "train_group": {"cld": {"Classification_binary": 2.471592529757961, "Classification": 0.017410958893373386, "Regression_pt": 0.14475292399212406, "Regression_eta": 0.013973752593246845, "Regression_sin_phi": 0.007746694687832225, "Regression_cos_phi": 0.008160829052536024, "Regression_energy": 0.14963558329655244, "Total": 5.385180242784746, "Total_unweighted": 2.8132732722736256}, "clic": {"Classification_binary": 2.3261854998365847, "Classification": 0.018661757992457843, "Regression_pt": 0.12564154475659667, "Regression_eta": 0.010703171917075614, "Regression_sin_phi": 0.008899445944566079, "Regression_cos_phi": 0.00923963860207109, "Regression_energy": 0.12669843595898794, "Total": 4.926303206089728, "Total_unweighted": 2.62602949500834}, "colliderml": {"Classification_binary": 2.3409784749463514, "Classification": 0.021497552722200258, "Regression_pt": 0.28470626285484246, "Regression_eta": 0.059077030489663104, "Regression_sin_phi": 0.06599983633370013, "Regression_cos_phi": 0.06601489668672865, "Regression_energy": 0.3232310384780437, "Total": 9.489488739151138, "Total_unweighted": 3.1615050925115296}}, "valid": {"Classification_binary": 2.3905331698330965, "Classification": 0.018627981706099077, "Regression_pt": 0.15752797560258344, "Regression_eta": 0.01873010071841153, "Regression_sin_phi": 0.016142732446843926, "Regression_cos_phi": 0.01638789393685081, "Regression_energy": 0.16518891074440695, "Total": 5.776715087890625, "group/cld/Classification": 0.017056787267644355, "group/cld/Classification_binary": 2.4422394772793385, "group/cld/Regression_cos_phi": 0.007934643867168021, "group/cld/Regression_energy": 0.14887711342344892, "group/cld/Regression_eta": 0.013219768696642936, "group/cld/Regression_pt": 0.14454600151548994, "group/cld/Regression_sin_phi": 0.007283440929778079, "group/cld/Total": 5.33124785727643, "group/cld/Total_unweighted": 2.781157232979511, "group/clic/Classification": 0.019325525202649706, "group/clic/Classification_binary": 2.3607739387674536, "group/clic/Regression_cos_phi": 0.008627083073271083, "group/clic/Regression_energy": 0.13099786068530792, "group/clic/Regression_eta": 0.01111565498595542, "group/clic/Regression_pt": 0.12800735108395841, "group/clic/Regression_sin_phi": 0.008488979111326502, "group/clic/Total": 5.011360980094747, "group/clic/Total_unweighted": 2.6673363929099225, "group/colliderml/Classification": 0.0211943369358778, "group/colliderml/Classification_binary": 2.3260560035705566, "group/colliderml/Regression_cos_phi": 0.06401665508747101, "group/colliderml/Regression_energy": 0.31354111433029175, "group/colliderml/Regression_eta": 0.05728411674499512, "group/colliderml/Regression_pt": 0.2823793590068817, "group/colliderml/Regression_sin_phi": 0.0646497830748558, "group/colliderml/Total": 9.333490371704102, "group/colliderml/Total_unweighted": 3.12912136875093, "Total_unweighted": 2.7831387649882924, "metrics/particle/class_1/efficiency": 0.957108199596405, "metrics/particle/class_1/pid_accuracy": 0.9829041957855225, "metrics/particle/class_2/efficiency": 0.27660036087036133, "metrics/particle/class_2/pid_accuracy": 0.6961846351623535, "metrics/particle/class_3/efficiency": 0.6545678377151489, "metrics/particle/class_3/pid_accuracy": 0.9754759669303894, "metrics/particle/class_4/efficiency": 0.8986432552337646, "metrics/particle/class_4/pid_accuracy": 0.4271758496761322, "metrics/particle/class_5/efficiency": 0.9754689931869507, "metrics/particle/class_5/pid_accuracy": 0.6227810382843018, "metrics/particle/count/bias_mean": -0.9828571677207947, "metrics/particle/count/mae": 4.555428504943848, "metrics/particle/count/prediction_mean": 91.17142486572266, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.048857443034648895, "metrics/particle/event/energy_response_mean": 0.9953969717025757, "metrics/particle/event/met_abs_error": 5.71353006362915, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0399189330637455, "metrics/particle/event/scalar_pt_response_mean": 1.0020110607147217, "metrics/particle/event/vector_pt_closure": 0.04186464101076126, "metrics/particle/matched/delta_eta_abs_mean": 0.012625513598322868, "metrics/particle/matched/delta_phi_abs_mean": 0.013173420913517475, "metrics/particle/matched/delta_r_mean": 0.020421180874109268, "metrics/particle/matched/energy_relative_abs_mean": 0.08852575719356537, "metrics/particle/matched/pid_accuracy": 0.9568856358528137, "metrics/particle/matched/pt_relative_abs_mean": 0.08398035168647766, "metrics/particle/matching/duplicate_fraction": 0.03274835646152496, "metrics/particle/matching/efficiency": 0.725292980670929, "metrics/particle/matching/f1": 0.7291814684867859, "metrics/particle/matching/purity": 0.7331118583679199, "metrics/particle/matching/target_coverage_dr0p05": 0.6658895015716553, "metrics/particle/matching/target_coverage_dr0p10": 0.7588205933570862, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6532151103019714, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.807223916053772}, "step_train_time": 17.40044379234314, "step_valid_time": 12.420349622960202, "step_total_time": 29477.435631752014, "elapsed_training_loop_seconds": 29477.435631752014, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9992426832381593, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0753258743515135, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1426, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1420, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9596231493943472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9635135135135136, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.961564396493594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03648648648648649, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.955585464333782, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9594594594594594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9575185434929198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9596231493943472, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0006664721379976, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08774118111902274, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1115, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1057, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.945438282647585, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9479820627802691, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9467084639498433, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05201793721973094, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9418604651162791, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9443946188340807, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9431258396775638, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.945438282647585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007918793716903, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08780162786338419, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 571, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 571, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9092356687898089, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9106858054226475, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9099601593625498, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08931419457735247, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9092356687898089, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9106858054226475, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9099601593625498, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9092356687898089, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9945811255010795, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.054014699515095854, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1424, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1421, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9544235924932976, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9512358049432198, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9528270324523251, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04876419505678023, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9524128686327078, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9492317969271877, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9508196721311475, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9544235924932976, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9960713851247887, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07215498148071631, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1101, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1035, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1033, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9315931593159316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9400544959128065, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9358047016274864, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05994550408719346, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9297929792979298, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9382379654859219, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.933996383363472, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9315931593159316, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9992704807892014, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07345673070721415, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 582, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 578, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9150943396226415, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9165354330708662, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9158143194335169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08346456692913386, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9088050314465409, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9102362204724409, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9095200629425649, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9150943396226415, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9840131301701265, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1363847440898368, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3585, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2712, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2677, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7645897941922751, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7564853556485356, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7605159842961301, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24351464435146444, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7547223005356639, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7467224546722455, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.750701065619742, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7645897941922751, "step_test_time": 35.72906987997703, "step_plot_time": 26.550330794067122, "step_validation_cycle_time": 75.43413196399342} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_240000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_240000.json new file mode 100644 index 0000000000000000000000000000000000000000..b045e095be9e089d962ffda8c6ab90f87d4be694 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_240000.json @@ -0,0 +1 @@ +{"epoch": 0.6091138662233672, "train": {"Classification_binary": 2.281369209289551, "Classification": 0.021973058581352234, "Regression_pt": 0.27767324447631836, "Regression_eta": 0.07220476120710373, "Regression_sin_phi": 0.06550617516040802, "Regression_cos_phi": 0.0658634677529335, "Regression_energy": 0.3299446702003479, "Total": 9.480010986328125, "Total_unweighted": 3.1145345866680145}, "train_group": {"cld": {"Classification_binary": 2.4477000469202035, "Classification": 0.017107900679558963, "Regression_pt": 0.14083273171841562, "Regression_eta": 0.013747782445744844, "Regression_sin_phi": 0.007535120017138553, "Regression_cos_phi": 0.007959492651426845, "Regression_energy": 0.1462052559575161, "Total": 5.289825681451562, "Total_unweighted": 2.7810883303900047}, "clic": {"Classification_binary": 2.313284163360481, "Classification": 0.0184753290950894, "Regression_pt": 0.12731674722991548, "Regression_eta": 0.010780835948610405, "Regression_sin_phi": 0.009231621535864097, "Regression_cos_phi": 0.009308688800541443, "Regression_energy": 0.12795601791895186, "Total": 4.940958071041393, "Total_unweighted": 2.616353403889454}, "colliderml": {"Classification_binary": 2.3340627433297163, "Classification": 0.02158751736902548, "Regression_pt": 0.28432561657921285, "Regression_eta": 0.05885537420427371, "Regression_sin_phi": 0.06591731884299282, "Regression_cos_phi": 0.06563978883142244, "Regression_energy": 0.3217474666541208, "Total": 9.46358974822267, "Total_unweighted": 3.1521358258107646}}, "valid": {"Classification_binary": 2.372601873224432, "Classification": 0.0184075420553034, "Regression_pt": 0.1581077228892933, "Regression_eta": 0.018955297903581098, "Regression_sin_phi": 0.016244621710343795, "Regression_cos_phi": 0.016316870125857266, "Regression_energy": 0.16574410525235264, "Total": 5.7668690074573865, "group/cld/Classification": 0.01685039540554615, "group/cld/Classification_binary": 2.4210774847801697, "group/cld/Regression_cos_phi": 0.007873632806412716, "group/cld/Regression_energy": 0.1503868001572629, "group/cld/Regression_eta": 0.01368284986374226, "group/cld/Regression_pt": 0.14587639747781955, "group/cld/Regression_sin_phi": 0.007406335561833483, "group/cld/Total": 5.333766856092088, "group/cld/Total_unweighted": 2.7631538960527866, "group/clic/Classification": 0.0190463865056951, "group/clic/Classification_binary": 2.342479299991689, "group/clic/Regression_cos_phi": 0.008727793997906625, "group/clic/Regression_energy": 0.131137766736619, "group/clic/Regression_eta": 0.011287405135783744, "group/clic/Regression_pt": 0.12836303102209212, "group/clic/Regression_sin_phi": 0.008597547703600944, "group/clic/Total": 4.9972498467627995, "group/clic/Total_unweighted": 2.6496392310933863, "group/colliderml/Classification": 0.021105043590068817, "group/colliderml/Classification_binary": 2.318690299987793, "group/colliderml/Regression_cos_phi": 0.06341178715229034, "group/colliderml/Regression_energy": 0.31251218914985657, "group/colliderml/Regression_eta": 0.05696754902601242, "group/colliderml/Regression_pt": 0.28141215443611145, "group/colliderml/Regression_sin_phi": 0.06467039883136749, "group/colliderml/Total": 9.299867630004883, "group/colliderml/Total_unweighted": 3.1187694221735, "Total_unweighted": 2.7663780331611636, "metrics/particle/class_1/efficiency": 0.9576180577278137, "metrics/particle/class_1/pid_accuracy": 0.9800735712051392, "metrics/particle/class_2/efficiency": 0.2858942151069641, "metrics/particle/class_2/pid_accuracy": 0.6775026321411133, "metrics/particle/class_3/efficiency": 0.6514523029327393, "metrics/particle/class_3/pid_accuracy": 0.9766668677330017, "metrics/particle/class_4/efficiency": 0.8850758075714111, "metrics/particle/class_4/pid_accuracy": 0.44454464316368103, "metrics/particle/class_5/efficiency": 0.98124098777771, "metrics/particle/class_5/pid_accuracy": 0.6926470398902893, "metrics/particle/count/bias_mean": -0.7405714392662048, "metrics/particle/count/mae": 4.478857040405273, "metrics/particle/count/prediction_mean": 91.41371154785156, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04616523161530495, "metrics/particle/event/energy_response_mean": 1.0021990537643433, "metrics/particle/event/met_abs_error": 5.5786919593811035, "metrics/particle/event/scalar_pt_relative_abs_error": 0.038510460406541824, "metrics/particle/event/scalar_pt_response_mean": 1.0084655284881592, "metrics/particle/event/vector_pt_closure": 0.04217365384101868, "metrics/particle/matched/delta_eta_abs_mean": 0.013493199832737446, "metrics/particle/matched/delta_phi_abs_mean": 0.014048213139176369, "metrics/particle/matched/delta_r_mean": 0.021771838888525963, "metrics/particle/matched/energy_relative_abs_mean": 0.08982962369918823, "metrics/particle/matched/pid_accuracy": 0.9549406170845032, "metrics/particle/matched/pt_relative_abs_mean": 0.08387351036071777, "metrics/particle/matching/duplicate_fraction": 0.03441809117794037, "metrics/particle/matching/efficiency": 0.725361168384552, "metrics/particle/matching/f1": 0.7282875180244446, "metrics/particle/matching/purity": 0.7312375903129578, "metrics/particle/matching/target_coverage_dr0p05": 0.6514726877212524, "metrics/particle/matching/target_coverage_dr0p10": 0.7590624690055847, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6524834036827087, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8075401782989502}, "step_train_time": 7.408073425292969, "step_valid_time": 12.502938461955637, "step_total_time": 30814.572855710983, "elapsed_training_loop_seconds": 30814.572855710983, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040094901714316, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07709360780704677, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9576043068640646, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9601889338731444, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9588948787061995, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0398110661268556, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9542395693135935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9568151147098516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9555256064690026, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9576043068640646, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003966972141242, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08755564894435741, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1061, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9490161001788909, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9481680071492404, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9485918641037103, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05183199285075961, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9428060768543343, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9432275368797497, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9490161001788909, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0099658750017113, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09155290893003976, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 566, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 563, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9012738853503185, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8998410174880763, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9005568814638027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10015898251192369, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8964968152866242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8950715421303657, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8957836117740652, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9012738853503185, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999573482431067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05779027857345587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1500, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9591152815013405, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9565508021390374, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9571045576407506, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.952, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9545454545454546, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9591152815013405, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029543520835702, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07918727005556092, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1097, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1039, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1036, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9351935193519352, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9471285323609845, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9411231884057971, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0528714676390155, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9324932493249325, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9443938012762079, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9384057971014492, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9351935193519352, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0049857480382898, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07555856812528294, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 634, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 577, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 576, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9072327044025157, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9100946372239748, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9086614173228347, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08990536277602523, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9056603773584906, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9085173501577287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9070866141732283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9072327044025157, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9932894042684829, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13212385128201498, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3580, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2661, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2628, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7502114462926417, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7432960893854749, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7467377578223656, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.25670391061452513, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7409078094164082, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7340782122905027, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7374771993826295, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7502114462926417, "step_test_time": 35.89322185504716, "step_plot_time": 26.539222801919095, "step_validation_cycle_time": 75.62865111394785} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_250000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_250000.json new file mode 100644 index 0000000000000000000000000000000000000000..b92e55813b9cc8cff2fa63076398bdde0986b3fa --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_250000.json @@ -0,0 +1 @@ +{"epoch": 0.6344936106493407, "train": {"Classification_binary": 2.3237075805664062, "Classification": 0.019380822777748108, "Regression_pt": 0.10442976653575897, "Regression_eta": 0.013428360223770142, "Regression_sin_phi": 0.006085023283958435, "Regression_cos_phi": 0.00976883340626955, "Regression_energy": 0.10391481965780258, "Total": 4.563511848449707, "Total_unweighted": 2.580715206451714}, "train_group": {"cld": {"Classification_binary": 2.4195041228197294, "Classification": 0.017182173112596937, "Regression_pt": 0.14140516671234976, "Regression_eta": 0.015371046788867809, "Regression_sin_phi": 0.0075731679963081805, "Regression_cos_phi": 0.007732259083823663, "Regression_energy": 0.1468257728718712, "Total": 5.279638015581463, "Total_unweighted": 2.755593709385547}, "clic": {"Classification_binary": 2.2817818656698003, "Classification": 0.01795633526815398, "Regression_pt": 0.12415477250878876, "Regression_eta": 0.01040402759431495, "Regression_sin_phi": 0.00902554316672723, "Regression_cos_phi": 0.009062501862673118, "Regression_energy": 0.12486664926236098, "Total": 4.843424121896784, "Total_unweighted": 2.5772516953328193}, "colliderml": {"Classification_binary": 2.323298965488468, "Classification": 0.02131165095322483, "Regression_pt": 0.2814320546012741, "Regression_eta": 0.05905563888830823, "Regression_sin_phi": 0.06576675731498557, "Regression_cos_phi": 0.06578365643998166, "Regression_energy": 0.31935767328237985, "Total": 9.406001822726505, "Total_unweighted": 3.1360063969686225}}, "valid": {"Classification_binary": 2.364140458540483, "Classification": 0.018171656131744384, "Regression_pt": 0.15679498152299362, "Regression_eta": 0.018623540618202903, "Regression_sin_phi": 0.015936661850322376, "Regression_cos_phi": 0.016089830615303734, "Regression_energy": 0.16401521509343928, "Total": 5.722568581321022, "group/cld/Classification": 0.016702974096257636, "group/cld/Classification_binary": 2.4076995849609375, "group/cld/Regression_cos_phi": 0.007658984432829187, "group/cld/Regression_energy": 0.146952933453499, "group/cld/Regression_eta": 0.01328220139158533, "group/cld/Regression_pt": 0.1432635429057669, "group/cld/Regression_sin_phi": 0.007176217246562876, "group/cld/Total": 5.262497922207447, "group/cld/Total_unweighted": 2.742736438487438, "group/clic/Classification": 0.018673210702043898, "group/clic/Classification_binary": 2.3387451171875, "group/clic/Regression_cos_phi": 0.008371446995024985, "group/clic/Regression_energy": 0.13087697739296772, "group/clic/Regression_eta": 0.01094050610319097, "group/clic/Regression_pt": 0.12810489979196102, "group/clic/Regression_sin_phi": 0.008296303292538257, "group/clic/Total": 4.974250631129488, "group/clic/Total_unweighted": 2.644008461465227, "group/colliderml/Classification": 0.021012593060731888, "group/colliderml/Classification_binary": 2.3107855319976807, "group/colliderml/Regression_cos_phi": 0.06352821737527847, "group/colliderml/Regression_energy": 0.3114793598651886, "group/colliderml/Regression_eta": 0.056882619857788086, "group/colliderml/Regression_pt": 0.28082072734832764, "group/colliderml/Regression_sin_phi": 0.06411401182413101, "group/colliderml/Total": 9.272205352783203, "group/colliderml/Total_unweighted": 3.1086230613291264, "Total_unweighted": 2.753772344372489, "metrics/particle/class_1/efficiency": 0.961388111114502, "metrics/particle/class_1/pid_accuracy": 0.9821928143501282, "metrics/particle/class_2/efficiency": 0.2900199890136719, "metrics/particle/class_2/pid_accuracy": 0.7000598907470703, "metrics/particle/class_3/efficiency": 0.6539111733436584, "metrics/particle/class_3/pid_accuracy": 0.9761136770248413, "metrics/particle/class_4/efficiency": 0.8930566906929016, "metrics/particle/class_4/pid_accuracy": 0.4584450423717499, "metrics/particle/class_5/efficiency": 0.98124098777771, "metrics/particle/class_5/pid_accuracy": 0.6617646813392639, "metrics/particle/count/bias_mean": -0.24171428382396698, "metrics/particle/count/mae": 4.405142784118652, "metrics/particle/count/prediction_mean": 91.9125747680664, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04808761924505234, "metrics/particle/event/energy_response_mean": 1.002972960472107, "metrics/particle/event/met_abs_error": 5.763416290283203, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03841711953282356, "metrics/particle/event/scalar_pt_response_mean": 1.0106104612350464, "metrics/particle/event/vector_pt_closure": 0.041188377887010574, "metrics/particle/matched/delta_eta_abs_mean": 0.012560033239424229, "metrics/particle/matched/delta_phi_abs_mean": 0.01316679734736681, "metrics/particle/matched/delta_r_mean": 0.020302867516875267, "metrics/particle/matched/energy_relative_abs_mean": 0.09171280264854431, "metrics/particle/matched/pid_accuracy": 0.956895112991333, "metrics/particle/matched/pt_relative_abs_mean": 0.08539889007806778, "metrics/particle/matching/duplicate_fraction": 0.03457944467663765, "metrics/particle/matching/efficiency": 0.7286165952682495, "metrics/particle/matching/f1": 0.7295734286308289, "metrics/particle/matching/purity": 0.7305327653884888, "metrics/particle/matching/target_coverage_dr0p05": 0.6769393086433411, "metrics/particle/matching/target_coverage_dr0p10": 0.766825795173645, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6471755504608154, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8057047128677368}, "step_train_time": 15.537895917892456, "step_valid_time": 12.497976796003059, "step_total_time": 32166.880239486694, "elapsed_training_loop_seconds": 32166.880239486694, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0079475467119425, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07465817659515683, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1490, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9577181208053691, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.959005376344086, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042281879194630875, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9576043068640646, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9550335570469799, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9563172043010753, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9602960969044414, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058074490957747, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08277530554744428, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1122, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1068, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1063, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552772808586762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9518716577540107, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9535714285714286, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0481283422459893, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9508050089445438, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9474153297682709, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9491071428571428, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552772808586762, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0129237289345916, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08395058685156909, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 577, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 576, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9187898089171974, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9173290937996821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9180588703261734, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08267090620031796, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9171974522292994, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9157392686804452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9164677804295943, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9187898089171974, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0046352219395958, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05865978940667582, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1495, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1429, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.957774798927614, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9558528428093646, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9568128557080683, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04414715719063545, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9571045576407506, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9551839464882943, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9561432875795112, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.957774798927614, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00511635355048, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07526255885580935, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1104, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1049, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1047, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9441944194419442, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501811594202898, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9471783295711061, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049818840579710144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9423942394239424, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9483695652173914, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9453724604966139, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9441944194419442, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0093857888390265, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0761137710478631, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 638, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 582, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 579, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9150943396226415, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9122257053291536, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9136577708006279, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0877742946708464, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.910377358490566, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9075235109717869, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9089481946624803, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9150943396226415, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0000808831521444, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13266342870690118, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3665, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2680, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2644, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7555680857062306, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7312414733969986, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7432057681641708, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.26875852660300137, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7454186636594305, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.721418826739427, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.733222407099279, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7555680857062306, "step_test_time": 35.923707096022554, "step_plot_time": 26.445214340928942, "step_validation_cycle_time": 75.57741022889968} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_260000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_260000.json new file mode 100644 index 0000000000000000000000000000000000000000..c56028197eef39e7aa4a2deceabe5a01e08f3b8c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_260000.json @@ -0,0 +1 @@ +{"epoch": 0.6598733550753144, "train": {"Classification_binary": 2.1665544509887695, "Classification": 0.02057608589529991, "Regression_pt": 0.11460765451192856, "Regression_eta": 0.008077679201960564, "Regression_sin_phi": 0.007758817635476589, "Regression_cos_phi": 0.006721808575093746, "Regression_energy": 0.11039145290851593, "Total": 4.5180439949035645, "Total_unweighted": 2.434687949717045}, "train_group": {"cld": {"Classification_binary": 2.39925995173755, "Classification": 0.016664178256090876, "Regression_pt": 0.13909445123540032, "Regression_eta": 0.013870096160658263, "Regression_sin_phi": 0.007469458818748906, "Regression_cos_phi": 0.007734090222644287, "Regression_energy": 0.1443119487157455, "Total": 5.203121344964425, "Total_unweighted": 2.728404175146838}, "clic": {"Classification_binary": 2.2658354854155442, "Classification": 0.01794210933762307, "Regression_pt": 0.12024621422330063, "Regression_eta": 0.01028066113504754, "Regression_sin_phi": 0.009135241185950707, "Regression_cos_phi": 0.008819711803804392, "Regression_energy": 0.1215312248709316, "Total": 4.76470699495898, "Total_unweighted": 2.5537906479722023}, "colliderml": {"Classification_binary": 2.3106277332649574, "Classification": 0.021215954347997456, "Regression_pt": 0.28006560375561584, "Regression_eta": 0.05869755260758214, "Regression_sin_phi": 0.065082076702032, "Regression_cos_phi": 0.06518505327962898, "Regression_energy": 0.31715366781294885, "Total": 9.346570000634179, "Total_unweighted": 3.118027641770763}}, "valid": {"Classification_binary": 2.3396908846768465, "Classification": 0.018077393011613326, "Regression_pt": 0.15548832633278586, "Regression_eta": 0.01856736053120006, "Regression_sin_phi": 0.015780924667011607, "Regression_cos_phi": 0.015969456325877797, "Regression_energy": 0.162956411188299, "Total": 5.67337646484375, "group/cld/Classification": 0.016447130669938758, "group/cld/Classification_binary": 2.372148554375831, "group/cld/Regression_cos_phi": 0.007558134007961192, "group/cld/Regression_energy": 0.14590215682983398, "group/cld/Regression_eta": 0.012885944640382807, "group/cld/Regression_pt": 0.1419140227297519, "group/cld/Regression_sin_phi": 0.006867827887230731, "group/cld/Total": 5.196009534470578, "group/cld/Total_unweighted": 2.7037237711409303, "group/clic/Classification": 0.01873369039373195, "group/clic/Classification_binary": 2.316855734967171, "group/clic/Regression_cos_phi": 0.008279672962553958, "group/clic/Regression_energy": 0.12948734202283493, "group/clic/Regression_eta": 0.01116221382262859, "group/clic/Regression_pt": 0.12646270305552382, "group/clic/Regression_sin_phi": 0.00822406309716245, "group/clic/Total": 4.9272425225440495, "group/clic/Total_unweighted": 2.619205420321607, "group/colliderml/Classification": 0.020938390865921974, "group/colliderml/Classification_binary": 2.311424493789673, "group/colliderml/Regression_cos_phi": 0.06326644122600555, "group/colliderml/Regression_energy": 0.3113686442375183, "group/colliderml/Regression_eta": 0.057009100914001465, "group/colliderml/Regression_pt": 0.28062552213668823, "group/colliderml/Regression_sin_phi": 0.06416144967079163, "group/colliderml/Total": 9.267407417297363, "group/colliderml/Total_unweighted": 3.1087940428406, "Total_unweighted": 2.726530756733634, "metrics/particle/class_1/efficiency": 0.9547750949859619, "metrics/particle/class_1/pid_accuracy": 0.9829757213592529, "metrics/particle/class_2/efficiency": 0.2863719165325165, "metrics/particle/class_2/pid_accuracy": 0.701546847820282, "metrics/particle/class_3/efficiency": 0.6588848233222961, "metrics/particle/class_3/pid_accuracy": 0.9765059947967529, "metrics/particle/class_4/efficiency": 0.8994413614273071, "metrics/particle/class_4/pid_accuracy": 0.432120680809021, "metrics/particle/class_5/efficiency": 0.9769119620323181, "metrics/particle/class_5/pid_accuracy": 0.673559844493866, "metrics/particle/count/bias_mean": -1.4279999732971191, "metrics/particle/count/mae": 4.4965715408325195, "metrics/particle/count/prediction_mean": 90.72628784179688, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04980788007378578, "metrics/particle/event/energy_response_mean": 1.0069499015808105, "metrics/particle/event/met_abs_error": 5.710040092468262, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04092388227581978, "metrics/particle/event/scalar_pt_response_mean": 1.015306830406189, "metrics/particle/event/vector_pt_closure": 0.04128444567322731, "metrics/particle/matched/delta_eta_abs_mean": 0.012361278757452965, "metrics/particle/matched/delta_phi_abs_mean": 0.013230576179921627, "metrics/particle/matched/delta_r_mean": 0.020205926150083542, "metrics/particle/matched/energy_relative_abs_mean": 0.08976903557777405, "metrics/particle/matched/pid_accuracy": 0.9574872255325317, "metrics/particle/matched/pt_relative_abs_mean": 0.08476332575082779, "metrics/particle/matching/duplicate_fraction": 0.034439537674188614, "metrics/particle/matching/efficiency": 0.7276802659034729, "metrics/particle/matching/f1": 0.7333623170852661, "metrics/particle/matching/purity": 0.7391337156295776, "metrics/particle/matching/target_coverage_dr0p05": 0.6748806238174438, "metrics/particle/matching/target_coverage_dr0p10": 0.7640540599822998, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6495628356933594, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8060891628265381}, "step_train_time": 15.001373767852783, "step_valid_time": 12.629103281069547, "step_total_time": 33496.78947377205, "elapsed_training_loop_seconds": 33496.78947377205, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0164717957033966, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08117212413375063, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1424, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1419, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9582772543741588, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9608636977058029, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9595687331536388, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03913630229419703, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9549125168236877, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9574898785425101, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.956199460916442, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9582772543741588, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.015196236405193, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08836007969768267, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1062, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9499105545617174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9490616621983914, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9494859186410372, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05093833780160858, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9428060768543343, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9432275368797497, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9499105545617174, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0208173883857357, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0789909590724095, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 576, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 575, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9171974522292994, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9157392686804452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9164677804295943, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08426073131955485, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9156050955414012, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9141494435612083, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9148766905330151, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9171974522292994, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0067628649461258, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.054072206113444765, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1494, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1425, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1420, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9550938337801609, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9538152610441767, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9544541192230409, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04618473895582329, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9517426273458445, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9504685408299867, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9511051574012056, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9550938337801609, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0081639683836343, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07305373069054066, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1097, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1040, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1038, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9360936093609361, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9480401093892434, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9420289855072463, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05195989061075661, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9342934293429342, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9462169553327257, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9402173913043478, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9360936093609361, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0099409759687132, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0695639072003087, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 638, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 574, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 571, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9025157232704403, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8996865203761756, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9010989010989011, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10031347962382445, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8977987421383647, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8949843260188087, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8963893249607535, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9025157232704403, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.996007178233048, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13496161010831975, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3564, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2671, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2647, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7530307301945306, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.749438832772166, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7512304879763746, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2505611672278339, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7462644488299972, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7427048260381593, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7444803825059767, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7530307301945306, "step_test_time": 36.17676664108876, "step_plot_time": 26.683805864071473, "step_validation_cycle_time": 76.19867184397299} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_270000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_270000.json new file mode 100644 index 0000000000000000000000000000000000000000..44e76529a32b48fbe836d5f6903307e477c965cc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_270000.json @@ -0,0 +1 @@ +{"epoch": 0.685253099501288, "train": {"Classification_binary": 2.259763717651367, "Classification": 0.02107962965965271, "Regression_pt": 0.25562313199043274, "Regression_eta": 0.06456641107797623, "Regression_sin_phi": 0.06339780241250992, "Regression_cos_phi": 0.0635836124420166, "Regression_energy": 0.29335546493530273, "Total": 8.89234447479248, "Total_unweighted": 3.021369770169258}, "train_group": {"cld": {"Classification_binary": 2.391541750581415, "Classification": 0.016849967158838613, "Regression_pt": 0.13926766683031488, "Regression_eta": 0.013868801869793205, "Regression_sin_phi": 0.007406110327841865, "Regression_cos_phi": 0.0074525884319063245, "Regression_energy": 0.1442341723286354, "Total": 5.194989717758454, "Total_unweighted": 2.7206210575287453}, "clic": {"Classification_binary": 2.26151869211111, "Classification": 0.01799608282073661, "Regression_pt": 0.12464709018354302, "Regression_eta": 0.010450484183569392, "Regression_sin_phi": 0.00880691303882759, "Regression_cos_phi": 0.00851927966287768, "Regression_energy": 0.1256939416175132, "Total": 4.825087590260549, "Total_unweighted": 2.5576324836181774}, "colliderml": {"Classification_binary": 2.305558118991509, "Classification": 0.02112926129519404, "Regression_pt": 0.2806811589680746, "Regression_eta": 0.05832756022939425, "Regression_sin_phi": 0.06499998495651933, "Regression_cos_phi": 0.06505797772171969, "Regression_energy": 0.3176964741802501, "Total": 9.345248429361218, "Total_unweighted": 3.1134505363426612}}, "valid": {"Classification_binary": 2.337432861328125, "Classification": 0.017921351302753796, "Regression_pt": 0.1552570169622248, "Regression_eta": 0.018712722171436658, "Regression_sin_phi": 0.01594967408613725, "Regression_cos_phi": 0.01606424179944125, "Regression_energy": 0.1627440886064009, "Total": 5.669390314275568, "group/cld/Classification": 0.016416176836541358, "group/cld/Classification_binary": 2.3677752880339926, "group/cld/Regression_cos_phi": 0.007699603095967719, "group/cld/Regression_energy": 0.14591376324917407, "group/cld/Regression_eta": 0.013007661129565948, "group/cld/Regression_pt": 0.141717718002644, "group/cld/Regression_sin_phi": 0.007148230963564934, "group/cld/Total": 5.195190754342587, "group/cld/Total_unweighted": 2.6996784413114505, "group/clic/Classification": 0.0184473547529667, "group/clic/Classification_binary": 2.3169430671854223, "group/clic/Regression_cos_phi": 0.008405597920113422, "group/clic/Regression_energy": 0.12981345805716007, "group/clic/Regression_eta": 0.011355084307650302, "group/clic/Regression_pt": 0.12713399846503076, "group/clic/Regression_sin_phi": 0.008315982970785587, "group/clic/Total": 4.935962433510638, "group/clic/Total_unweighted": 2.6204145436591295, "group/colliderml/Classification": 0.020797640085220337, "group/colliderml/Classification_binary": 2.3084909915924072, "group/colliderml/Regression_cos_phi": 0.06313265115022659, "group/colliderml/Regression_energy": 0.30891653895378113, "group/colliderml/Regression_eta": 0.05708441138267517, "group/colliderml/Regression_pt": 0.2776399850845337, "group/colliderml/Regression_sin_phi": 0.0642278790473938, "group/colliderml/Total": 9.216790199279785, "group/colliderml/Total_unweighted": 3.100290097296238, "Total_unweighted": 2.7240819562565197, "metrics/particle/class_1/efficiency": 0.9618207216262817, "metrics/particle/class_1/pid_accuracy": 0.9814136624336243, "metrics/particle/class_2/efficiency": 0.29779380559921265, "metrics/particle/class_2/pid_accuracy": 0.7020562887191772, "metrics/particle/class_3/efficiency": 0.6639283299446106, "metrics/particle/class_3/pid_accuracy": 0.9741383194923401, "metrics/particle/class_4/efficiency": 0.916201114654541, "metrics/particle/class_4/pid_accuracy": 0.4608013927936554, "metrics/particle/class_5/efficiency": 0.9754689931869507, "metrics/particle/class_5/pid_accuracy": 0.6997041702270508, "metrics/particle/count/bias_mean": 1.2331428527832031, "metrics/particle/count/mae": 4.363428592681885, "metrics/particle/count/prediction_mean": 93.3874282836914, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.046134695410728455, "metrics/particle/event/energy_response_mean": 1.003599762916565, "metrics/particle/event/met_abs_error": 5.4581074714660645, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036633364856243134, "metrics/particle/event/scalar_pt_response_mean": 1.0046522617340088, "metrics/particle/event/vector_pt_closure": 0.03975091129541397, "metrics/particle/matched/delta_eta_abs_mean": 0.013045069761574268, "metrics/particle/matched/delta_phi_abs_mean": 0.014090284705162048, "metrics/particle/matched/delta_r_mean": 0.021421940997242928, "metrics/particle/matched/energy_relative_abs_mean": 0.09056894481182098, "metrics/particle/matched/pid_accuracy": 0.955670177936554, "metrics/particle/matched/pt_relative_abs_mean": 0.0852414220571518, "metrics/particle/matching/duplicate_fraction": 0.03727635368704796, "metrics/particle/matching/efficiency": 0.7345011234283447, "metrics/particle/matching/f1": 0.7296195030212402, "metrics/particle/matching/purity": 0.7248023748397827, "metrics/particle/matching/target_coverage_dr0p05": 0.6674644947052002, "metrics/particle/matching/target_coverage_dr0p10": 0.7698951959609985, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6570285558700562, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8166242837905884}, "step_train_time": 15.449043273925781, "step_valid_time": 12.706732155987993, "step_total_time": 34821.40870499611, "elapsed_training_loop_seconds": 34821.40870499611, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9995302547949896, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07135933744536138, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1476, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1419, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9549125168236877, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9613821138211383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9581363943281567, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03861788617886179, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9502018842530283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9566395663956639, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534098582039163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9549125168236877, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9992011812918915, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08022185901451617, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1120, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1061, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1058, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9490161001788909, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9473214285714285, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9481680071492404, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05267857142857143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9463327370304114, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9446428571428571, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9454870420017873, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9490161001788909, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003371776485083, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08269172120569801, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 570, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 569, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9076433121019108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.912, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9098164405426975, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.088, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9060509554140127, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9104, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9082202713487629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9076433121019108, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.99880769648352, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0539225152663626, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1437, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9631367292225201, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9624916275954454, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9628140703517588, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03750837240455459, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9597855227882037, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9591426657736102, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.959463986599665, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9631367292225201, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036982720291392, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07637853297657982, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1099, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1041, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1038, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936993699369937, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9472247497725205, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9420814479638009, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052775250227479524, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9342934293429342, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9444949954504095, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9393665158371041, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936993699369937, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001547026582664, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0677686356517263, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 637, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 578, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 574, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9088050314465409, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9073783359497645, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9080911233307148, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09262166405023547, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9025157232704403, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9010989010989011, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9018067556952082, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9088050314465409, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9944709443129971, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13580383068609314, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3629, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2688, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2653, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7578235128277417, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7406999173325985, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7491638795986622, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2593000826674015, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7479560191711305, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.731055387158997, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7394091415830546, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7578235128277417, "step_test_time": 36.08504684595391, "step_plot_time": 26.456966318073682, "step_validation_cycle_time": 75.94333516596816} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_280000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_280000.json new file mode 100644 index 0000000000000000000000000000000000000000..83a3942885ee115be3c799e008a4f1cfbfaeb50a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_280000.json @@ -0,0 +1 @@ +{"epoch": 0.7106328439272617, "train": {"Classification_binary": 2.260157823562622, "Classification": 0.016242414712905884, "Regression_pt": 0.12085043638944626, "Regression_eta": 0.004993793088942766, "Regression_sin_phi": 0.010200289078056812, "Regression_cos_phi": 0.011671825312077999, "Regression_energy": 0.11973719298839569, "Total": 4.746166229248047, "Total_unweighted": 2.5438537751324475}, "train_group": {"cld": {"Classification_binary": 2.353864214734403, "Classification": 0.016680332784635758, "Regression_pt": 0.13320484152900244, "Regression_eta": 0.015318224647829677, "Regression_sin_phi": 0.007311586456507177, "Regression_cos_phi": 0.007659985298363347, "Regression_energy": 0.13844635964421456, "Total": 5.067285093004832, "Total_unweighted": 2.672485545094956}, "clic": {"Classification_binary": 2.244130303193857, "Classification": 0.01801509123754215, "Regression_pt": 0.12265524572438306, "Regression_eta": 0.010807129768402996, "Regression_sin_phi": 0.009000200205608396, "Regression_cos_phi": 0.008869431189236205, "Regression_energy": 0.12350768640682146, "Total": 4.78267609178125, "Total_unweighted": 2.536985087725851}, "colliderml": {"Classification_binary": 2.3017799925875737, "Classification": 0.021076862647145003, "Regression_pt": 0.2764069193207824, "Regression_eta": 0.058303352411802824, "Regression_sin_phi": 0.0651121926871506, "Regression_cos_phi": 0.06477074754667712, "Regression_energy": 0.3131609606044786, "Total": 9.266836300984517, "Total_unweighted": 3.10061102780561}}, "valid": {"Classification_binary": 2.323729081587358, "Classification": 0.017925143241882324, "Regression_pt": 0.15242058147083629, "Regression_eta": 0.018680982156233355, "Regression_sin_phi": 0.015733219276775012, "Regression_cos_phi": 0.01598655635660345, "Regression_energy": 0.16004385514692826, "Total": 5.606382057883523, "group/cld/Classification": 0.01636459472331595, "group/cld/Classification_binary": 2.3613693561959774, "group/cld/Regression_cos_phi": 0.0074950526369378925, "group/cld/Regression_energy": 0.14182521941814016, "group/cld/Regression_eta": 0.012990512746445675, "group/cld/Regression_pt": 0.1375011586128397, "group/cld/Regression_sin_phi": 0.006964447650503605, "group/cld/Total": 5.115039906603225, "group/cld/Total_unweighted": 2.68451034198416, "group/clic/Classification": 0.01846988150414, "group/clic/Classification_binary": 2.2980658348570477, "group/clic/Regression_cos_phi": 0.008329409868159193, "group/clic/Regression_energy": 0.12868249162714532, "group/clic/Regression_eta": 0.011335678557132153, "group/clic/Regression_pt": 0.12559454491797914, "group/clic/Regression_sin_phi": 0.008131206035614014, "group/clic/Total": 4.891541176653923, "group/clic/Total_unweighted": 2.5986090473672174, "group/colliderml/Classification": 0.020909111946821213, "group/colliderml/Classification_binary": 2.288545846939087, "group/colliderml/Regression_cos_phi": 0.06342320889234543, "group/colliderml/Regression_energy": 0.3056851327419281, "group/colliderml/Regression_eta": 0.05697356164455414, "group/colliderml/Regression_pt": 0.2750478684902191, "group/colliderml/Regression_sin_phi": 0.06382236629724503, "group/colliderml/Total": 9.149538040161133, "group/colliderml/Total_unweighted": 3.0744070969522, "Total_unweighted": 2.7045194192366164, "metrics/particle/class_1/efficiency": 0.9578807353973389, "metrics/particle/class_1/pid_accuracy": 0.9830631613731384, "metrics/particle/class_2/efficiency": 0.28936854004859924, "metrics/particle/class_2/pid_accuracy": 0.7082395553588867, "metrics/particle/class_3/efficiency": 0.6531288027763367, "metrics/particle/class_3/pid_accuracy": 0.9758711457252502, "metrics/particle/class_4/efficiency": 0.9066240787506104, "metrics/particle/class_4/pid_accuracy": 0.39788731932640076, "metrics/particle/class_5/efficiency": 0.9769119620323181, "metrics/particle/class_5/pid_accuracy": 0.688330888748169, "metrics/particle/count/bias_mean": -1.2942856550216675, "metrics/particle/count/mae": 4.48628568649292, "metrics/particle/count/prediction_mean": 90.86000061035156, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04667963460087776, "metrics/particle/event/energy_response_mean": 1.0093152523040771, "metrics/particle/event/met_abs_error": 5.468076229095459, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03927776589989662, "metrics/particle/event/scalar_pt_response_mean": 1.018054723739624, "metrics/particle/event/vector_pt_closure": 0.03976213186979294, "metrics/particle/matched/delta_eta_abs_mean": 0.011757252737879753, "metrics/particle/matched/delta_phi_abs_mean": 0.012654921039938927, "metrics/particle/matched/delta_r_mean": 0.0192948617041111, "metrics/particle/matched/energy_relative_abs_mean": 0.0932115837931633, "metrics/particle/matched/pid_accuracy": 0.9572001099586487, "metrics/particle/matched/pt_relative_abs_mean": 0.08823765069246292, "metrics/particle/matching/duplicate_fraction": 0.03463413193821907, "metrics/particle/matching/efficiency": 0.7268555760383606, "metrics/particle/matching/f1": 0.7319959402084351, "metrics/particle/matching/purity": 0.7372094988822937, "metrics/particle/matching/target_coverage_dr0p05": 0.6770818829536438, "metrics/particle/matching/target_coverage_dr0p10": 0.7646307349205017, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6448378562927246, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8046382069587708}, "step_train_time": 15.271745681762695, "step_valid_time": 12.649403617950156, "step_total_time": 36157.685420036316, "elapsed_training_loop_seconds": 36157.685420036316, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0169451126599696, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0710885113083588, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1493, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1430, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9623149394347241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9578030810448761, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9600537092984223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04219691895512391, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9576043068640646, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9531145344943067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9553541456864719, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9623149394347241, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0175834638099825, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08498526729078715, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1121, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1065, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9536128456735058, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.954890576150067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0463871543264942, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9525939177101967, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9500446030330062, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9513175524787851, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561717352415027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.020970292384898, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08276152349180621, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 576, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 575, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9171974522292994, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9201277955271565, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9186602870813397, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07987220447284345, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9156050955414012, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9185303514376997, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9170653907496013, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9171974522292994, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0124562992888637, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.054357027510952505, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1496, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1428, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1424, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9571045576407506, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9545454545454546, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9558232931726908, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045454545454545456, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9544235924932976, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9518716577540107, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9531459170013387, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9571045576407506, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0143214362463773, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06582800776687037, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1102, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1035, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1034, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9315931593159316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9392014519056261, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9353818346136467, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.060798548094373864, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9306930693069307, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9382940108892922, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9344780840488025, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9315931593159316, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0148851271320587, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06611034538700766, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 641, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 586, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 583, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9213836477987422, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9141965678627145, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9177760375880971, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08580343213728549, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9166666666666666, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9095163806552262, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9130775254502741, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9213836477987422, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058336011551947, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13390185921903885, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3614, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2713, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2676, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.764871722582464, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7506917542888766, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7577154028766931, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24930824571112342, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7544403721454751, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.740453790813503, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7473816506074571, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.764871722582464, "step_test_time": 36.11422704102006, "step_plot_time": 26.67818361404352, "step_validation_cycle_time": 76.16996682994068} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_290000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_290000.json new file mode 100644 index 0000000000000000000000000000000000000000..3576bce619bd58674a9ebb58374443fcfbb2391c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_290000.json @@ -0,0 +1 @@ +{"epoch": 0.7360125883532352, "train": {"Classification_binary": 2.3835384845733643, "Classification": 0.014858522452414036, "Regression_pt": 0.10544602572917938, "Regression_eta": 0.011669241823256016, "Regression_sin_phi": 0.012182452715933323, "Regression_cos_phi": 0.012098435312509537, "Regression_energy": 0.108405202627182, "Total": 4.6935577392578125, "Total_unweighted": 2.6481983652338386}, "train_group": {"cld": {"Classification_binary": 2.35017154023454, "Classification": 0.01673755717572865, "Regression_pt": 0.1333276687396897, "Regression_eta": 0.014523477147420024, "Regression_sin_phi": 0.007248505611565557, "Regression_cos_phi": 0.007821842780554885, "Regression_energy": 0.1390250432374957, "Total": 5.065829094465788, "Total_unweighted": 2.6688556349269943}, "clic": {"Classification_binary": 2.235645874531683, "Classification": 0.01786684808962627, "Regression_pt": 0.12428387910306096, "Regression_eta": 0.010948758472655036, "Regression_sin_phi": 0.008890335886560737, "Regression_cos_phi": 0.008906977573928318, "Regression_energy": 0.1251363073905071, "Total": 4.798805889255273, "Total_unweighted": 2.5316789810480214}, "colliderml": {"Classification_binary": 2.2966464561027093, "Classification": 0.021007170386097812, "Regression_pt": 0.2775199455810381, "Regression_eta": 0.058723425833849575, "Regression_sin_phi": 0.06523738651572764, "Regression_cos_phi": 0.06517227054447741, "Regression_energy": 0.3145421898221826, "Total": 9.289523438290432, "Total_unweighted": 3.0988488447860822}}, "valid": {"Classification_binary": 2.3102046619762073, "Classification": 0.01773666035045277, "Regression_pt": 0.15232514468106356, "Regression_eta": 0.018416138128800826, "Regression_sin_phi": 0.015736835653131657, "Regression_cos_phi": 0.01589009870182384, "Regression_energy": 0.15966158780184658, "Total": 5.584213534268466, "group/cld/Classification": 0.016031422513596554, "group/cld/Classification_binary": 2.346013657590176, "group/cld/Regression_cos_phi": 0.007481758264785117, "group/cld/Regression_energy": 0.14039280059489798, "group/cld/Regression_eta": 0.01278403084328834, "group/cld/Regression_pt": 0.13645655043581698, "group/cld/Regression_sin_phi": 0.006931655584497654, "group/cld/Total": 5.073755791846742, "group/cld/Total_unweighted": 2.666091875827058, "group/clic/Classification": 0.018348490938227227, "group/clic/Classification_binary": 2.2791308139232878, "group/clic/Regression_cos_phi": 0.008289921790995497, "group/clic/Regression_energy": 0.12872741577473093, "group/clic/Regression_eta": 0.011005867034830946, "group/clic/Regression_pt": 0.12592343066601044, "group/clic/Regression_sin_phi": 0.008095559287578501, "group/clic/Total": 4.871657351230053, "group/clic/Total_unweighted": 2.579521499415662, "group/colliderml/Classification": 0.02094850316643715, "group/colliderml/Classification_binary": 2.2962934970855713, "group/colliderml/Regression_cos_phi": 0.06291510909795761, "group/colliderml/Regression_energy": 0.3071324825286865, "group/colliderml/Regression_eta": 0.05672813206911087, "group/colliderml/Regression_pt": 0.27649420499801636, "group/colliderml/Regression_sin_phi": 0.06404831260442734, "group/colliderml/Total": 9.1768159866333, "group/colliderml/Total_unweighted": 3.084560241550207, "Total_unweighted": 2.689971127293326, "metrics/particle/class_1/efficiency": 0.9590859413146973, "metrics/particle/class_1/pid_accuracy": 0.9826172590255737, "metrics/particle/class_2/efficiency": 0.3006601333618164, "metrics/particle/class_2/pid_accuracy": 0.7160190939903259, "metrics/particle/class_3/efficiency": 0.6631879210472107, "metrics/particle/class_3/pid_accuracy": 0.9743621945381165, "metrics/particle/class_4/efficiency": 0.910614550113678, "metrics/particle/class_4/pid_accuracy": 0.43558281660079956, "metrics/particle/class_5/efficiency": 0.9754689931869507, "metrics/particle/class_5/pid_accuracy": 0.6671597361564636, "metrics/particle/count/bias_mean": 0.1371428519487381, "metrics/particle/count/mae": 4.163428783416748, "metrics/particle/count/prediction_mean": 92.29142761230469, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04720783233642578, "metrics/particle/event/energy_response_mean": 1.0166388750076294, "metrics/particle/event/met_abs_error": 5.4552741050720215, "metrics/particle/event/scalar_pt_relative_abs_error": 0.038640573620796204, "metrics/particle/event/scalar_pt_response_mean": 1.0196117162704468, "metrics/particle/event/vector_pt_closure": 0.0411984957754612, "metrics/particle/matched/delta_eta_abs_mean": 0.012620396912097931, "metrics/particle/matched/delta_phi_abs_mean": 0.013683518394827843, "metrics/particle/matched/delta_r_mean": 0.0207822322845459, "metrics/particle/matched/energy_relative_abs_mean": 0.0918600782752037, "metrics/particle/matched/pid_accuracy": 0.9566206336021423, "metrics/particle/matched/pt_relative_abs_mean": 0.08692169934511185, "metrics/particle/matching/duplicate_fraction": 0.03699461370706558, "metrics/particle/matching/efficiency": 0.7334408164024353, "metrics/particle/matching/f1": 0.7328954935073853, "metrics/particle/matching/purity": 0.7323509454727173, "metrics/particle/matching/target_coverage_dr0p05": 0.676027774810791, "metrics/particle/matching/target_coverage_dr0p10": 0.770310640335083, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6519191265106201, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8122837543487549}, "step_train_time": 14.736348390579224, "step_valid_time": 12.774624670972116, "step_total_time": 37489.496235370636, "elapsed_training_loop_seconds": 37489.496235370636, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0119229845680033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07256557192647173, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1476, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9528936742934051, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.959349593495935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9561107359891965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04065040650406504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9502018842530283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9566395663956639, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534098582039163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9528936742934051, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0115012536081711, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08748935036537586, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1116, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1058, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9463327370304114, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9480286738351255, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9471799462846912, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05197132616487455, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9453405017921147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9444941808415398, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9463327370304114, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0153713999427625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09340363701065846, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 577, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 574, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9187898089171974, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9246794871794872, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9217252396166135, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07532051282051282, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9140127388535032, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9198717948717948, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9169329073482428, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9187898089171974, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0070003942354453, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05561225041022433, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9591152815013405, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9616935483870968, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9604026845637584, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038306451612903226, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9571045576407506, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9596774193548387, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9583892617449664, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9591152815013405, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0125266556055024, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07479317891396375, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1098, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1049, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1046, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9441944194419442, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9553734061930783, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9497510185604345, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044626593806921674, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9414941494149415, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9526411657559198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9470348574015391, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9441944194419442, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0101630335200462, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06924970247783346, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 586, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 585, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9213836477987422, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9228346456692913, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9221085759244689, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07716535433070866, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9198113207547169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9212598425196851, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9205350118017309, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9213836477987422, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032810715262475, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13186270463028238, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3659, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2685, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2648, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7569777276571751, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7338070511068598, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7452123230641132, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2661929488931402, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.746546377220186, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7236949986335064, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7349431029697474, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7569777276571751, "step_test_time": 39.20352931204252, "step_plot_time": 26.73083856701851, "step_validation_cycle_time": 79.4187637490686} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..1a5b615addb8e182e1f2a438aea156c2fa0fbc26 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 0.0761392332779209, "train": {"Classification_binary": 2.984788417816162, "Classification": 0.03085809201002121, "Regression_pt": 0.5185518264770508, "Regression_eta": 0.07250246405601501, "Regression_sin_phi": 0.08100474625825882, "Regression_cos_phi": 0.08401738107204437, "Regression_energy": 0.5843726396560669, "Total": 14.746761322021484, "Total_unweighted": 4.356095567345619}, "train_group": {"cld": {"Classification_binary": 3.957887555743839, "Classification": 0.030755636050133733, "Regression_pt": 0.25524725104953433, "Regression_eta": 0.025204472136110427, "Regression_sin_phi": 0.012918493613969904, "Regression_cos_phi": 0.013113161631476682, "Regression_energy": 0.2652315259337783, "Total": 9.084252102596981, "Total_unweighted": 4.560358096158843}, "clic": {"Classification_binary": 3.6479215586149656, "Classification": 0.033157958131697446, "Regression_pt": 0.2167325030128519, "Regression_eta": 0.019805312710436614, "Regression_sin_phi": 0.015441566700110206, "Regression_cos_phi": 0.015667531937192957, "Regression_energy": 0.21875312011521142, "Total": 8.153279073961505, "Total_unweighted": 4.167479551222466}, "colliderml": {"Classification_binary": 3.0481967354962927, "Classification": 0.03092846977608111, "Regression_pt": 0.40869069670488734, "Regression_eta": 0.06684994766722896, "Regression_sin_phi": 0.07770234788070896, "Regression_cos_phi": 0.07732037651146244, "Regression_energy": 0.4573307553986589, "Total": 12.74551635445235, "Total_unweighted": 4.167019329435321}}, "valid": {"Classification_binary": 3.6722578568892046, "Classification": 0.03192789121107622, "Regression_pt": 0.26453562649813567, "Regression_eta": 0.028252051093361593, "Regression_sin_phi": 0.02221495888449929, "Regression_cos_phi": 0.022635009072043678, "Regression_energy": 0.27523990977894175, "Total": 9.216435102982954, "group/cld/Classification": 0.030587168450051164, "group/cld/Classification_binary": 3.931691271193484, "group/cld/Regression_cos_phi": 0.013382418358579595, "group/cld/Regression_energy": 0.2737473426981175, "group/cld/Regression_eta": 0.024828385799489122, "group/cld/Regression_pt": 0.2658717378656915, "group/cld/Regression_sin_phi": 0.011848202411164629, "group/cld/Total": 9.20399767287234, "group/cld/Total_unweighted": 4.5519565267765785, "group/clic/Classification": 0.033779623660635444, "group/clic/Classification_binary": 3.631599913252161, "group/clic/Regression_cos_phi": 0.014224094279268955, "group/clic/Regression_energy": 0.2185015171132189, "group/clic/Regression_eta": 0.019386268676595486, "group/clic/Regression_pt": 0.21470075972536776, "group/clic/Regression_sin_phi": 0.014499403060750758, "group/clic/Total": 8.09251111619016, "group/clic/Total_unweighted": 4.146691579767999, "group/colliderml/Classification": 0.030426718294620514, "group/colliderml/Classification_binary": 3.029607057571411, "group/colliderml/Regression_cos_phi": 0.07452155649662018, "group/colliderml/Regression_energy": 0.44629329442977905, "group/colliderml/Regression_eta": 0.06435229629278183, "group/colliderml/Regression_pt": 0.4070011079311371, "group/colliderml/Regression_sin_phi": 0.07533180713653564, "group/colliderml/Total": 12.554498672485352, "group/colliderml/Total_unweighted": 4.127533838152885, "Total_unweighted": 4.317063303427263, "metrics/particle/class_1/efficiency": 0.9163795113563538, "metrics/particle/class_1/pid_accuracy": 0.975922703742981, "metrics/particle/class_2/efficiency": 0.1377573162317276, "metrics/particle/class_2/pid_accuracy": 0.2856242060661316, "metrics/particle/class_3/efficiency": 0.5103874206542969, "metrics/particle/class_3/pid_accuracy": 0.9590222239494324, "metrics/particle/class_4/efficiency": 0.8619313836097717, "metrics/particle/class_4/pid_accuracy": 0.011111111380159855, "metrics/particle/class_5/efficiency": 0.9610389471054077, "metrics/particle/class_5/pid_accuracy": 0.115615613758564, "metrics/particle/count/bias_mean": -3.1937143802642822, "metrics/particle/count/mae": 7.734285831451416, "metrics/particle/count/prediction_mean": 88.9605712890625, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.08929117023944855, "metrics/particle/event/energy_response_mean": 0.9836060404777527, "metrics/particle/event/met_abs_error": 13.518818855285645, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07878731191158295, "metrics/particle/event/scalar_pt_response_mean": 0.9917603135108948, "metrics/particle/event/vector_pt_closure": 0.08645007014274597, "metrics/particle/matched/delta_eta_abs_mean": 0.02499847114086151, "metrics/particle/matched/delta_phi_abs_mean": 0.019741961732506752, "metrics/particle/matched/delta_r_mean": 0.03539957478642464, "metrics/particle/matched/energy_relative_abs_mean": 0.10181347280740738, "metrics/particle/matched/pid_accuracy": 0.9320358633995056, "metrics/particle/matched/pt_relative_abs_mean": 0.09986110031604767, "metrics/particle/matching/duplicate_fraction": 0.040364593267440796, "metrics/particle/matching/efficiency": 0.624784529209137, "metrics/particle/matching/f1": 0.6358017325401306, "metrics/particle/matching/purity": 0.6472144722938538, "metrics/particle/matching/target_coverage_dr0p05": 0.48299744725227356, "metrics/particle/matching/target_coverage_dr0p10": 0.6598499417304993, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5724003314971924, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7334904074668884}, "step_train_time": 7.310272216796875, "step_valid_time": 12.45915045205038, "step_total_time": 3705.097484588623, "elapsed_training_loop_seconds": 3705.097484588623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9913869994411175, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1870390829581231, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1491, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1348, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1293, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9071332436069987, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9040912139503688, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9056096741686261, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09590878604963111, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.870121130551817, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8672032193158954, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.868659724554921, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9071332436069987, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9911626011769666, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20345682787799357, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 953, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 917, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8524150268336315, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8531781557743957, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8527964205816555, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1468218442256043, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8202146690518783, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8209489704565801, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8205816554809844, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8524150268336315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000030661580453, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18962024239035125, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 634, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 501, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 489, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7977707006369427, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7902208201892744, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7939778129952456, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.20977917981072555, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7786624203821656, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7712933753943217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7749603803486529, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7977707006369427, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9985762206310453, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14769916680540085, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1506, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1361, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1326, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9121983914209115, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9037184594953519, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9079386257505003, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09628154050464807, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8887399463806971, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8804780876494024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8845897264843229, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9121983914209115, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9998787660830691, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1729816796070961, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1091, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 962, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 941, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8658865886588659, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8817598533455545, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8737511353315168, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11824014665444546, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.846984698469847, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8625114573785518, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8546775658492279, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8658865886588659, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9963931368930319, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18116900101945566, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 650, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 509, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 499, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.800314465408805, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7830769230769231, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7916018662519441, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2169230769230769, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7845911949685535, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7676923076923077, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.776049766718507, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.800314465408805, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9902347471157624, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.30428845823820927, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3608, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2465, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2272, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.6949534818156188, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.6832039911308204, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.6890286512928022, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.3167960088691796, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6405413025091626, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6297117516629712, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6350803633822502, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.6949534818156188, "step_test_time": 33.49553478998132, "step_plot_time": 26.262970164068975, "step_validation_cycle_time": 72.75247834599577} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_300000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_300000.json new file mode 100644 index 0000000000000000000000000000000000000000..986ca63c9b114961209821b14b653672fe66059a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_300000.json @@ -0,0 +1 @@ +{"epoch": 0.7613923327792089, "train": {"Classification_binary": 2.2422642707824707, "Classification": 0.020792344585061073, "Regression_pt": 0.2651573121547699, "Regression_eta": 0.05266386270523071, "Regression_sin_phi": 0.06460520625114441, "Regression_cos_phi": 0.06059940531849861, "Regression_energy": 0.30506792664527893, "Total": 8.950763702392578, "Total_unweighted": 3.0111503284424543}, "train_group": {"cld": {"Classification_binary": 2.3335718496425732, "Classification": 0.016241682511572545, "Regression_pt": 0.13083171846719832, "Regression_eta": 0.012888625218720362, "Regression_sin_phi": 0.007261738496099797, "Regression_cos_phi": 0.0073984018652825745, "Regression_energy": 0.13540147517894482, "Total": 4.979786488386962, "Total_unweighted": 2.6435954913803914}, "clic": {"Classification_binary": 2.246365824977199, "Classification": 0.01782241781653316, "Regression_pt": 0.12056118041813911, "Regression_eta": 0.010403698594826791, "Regression_sin_phi": 0.008657827972666593, "Regression_cos_phi": 0.008937194816874447, "Regression_energy": 0.12128475028741825, "Total": 4.741384713857381, "Total_unweighted": 2.5340328948836572}, "colliderml": {"Classification_binary": 2.282816802670142, "Classification": 0.02088079843558594, "Regression_pt": 0.2760532011172015, "Regression_eta": 0.0577462150932786, "Regression_sin_phi": 0.06497955280997439, "Regression_cos_phi": 0.06479936722152961, "Regression_energy": 0.31220153023204406, "Total": 9.230913299286437, "Total_unweighted": 3.079477467579756}}, "valid": {"Classification_binary": 2.2954680009321735, "Classification": 0.017610079591924493, "Regression_pt": 0.15168963345614347, "Regression_eta": 0.018341430750760165, "Regression_sin_phi": 0.015618556196039373, "Regression_cos_phi": 0.015889517827467486, "Regression_energy": 0.15906963348388672, "Total": 5.5557373046875, "group/cld/Classification": 0.01604951315737785, "group/cld/Classification_binary": 2.3189031722697804, "group/cld/Regression_cos_phi": 0.007522740262619993, "group/cld/Regression_energy": 0.14159924933250914, "group/cld/Regression_eta": 0.012698045436372148, "group/cld/Regression_pt": 0.13723764013736806, "group/cld/Regression_sin_phi": 0.006834491136226248, "group/cld/Total": 5.061252999812998, "group/cld/Total_unweighted": 2.6408448517322536, "group/clic/Classification": 0.018176530269866296, "group/clic/Classification_binary": 2.2782417459690825, "group/clic/Regression_cos_phi": 0.008284885832603941, "group/clic/Regression_energy": 0.1270769809154754, "group/clic/Regression_eta": 0.010955941169819933, "group/clic/Regression_pt": 0.12438376406405835, "group/clic/Regression_sin_phi": 0.008097139444757016, "group/clic/Total": 4.842154644905253, "group/clic/Total_unweighted": 2.5752169876656628, "group/colliderml/Classification": 0.020530277863144875, "group/colliderml/Classification_binary": 2.277231216430664, "group/colliderml/Regression_cos_phi": 0.06280550360679626, "group/colliderml/Regression_energy": 0.3043671250343323, "group/colliderml/Regression_eta": 0.05661375820636749, "group/colliderml/Regression_pt": 0.27435338497161865, "group/colliderml/Regression_sin_phi": 0.06351591646671295, "group/colliderml/Total": 9.104437828063965, "group/colliderml/Total_unweighted": 3.0594171825796366, "Total_unweighted": 2.6736868522383954, "metrics/particle/class_1/efficiency": 0.9591322541236877, "metrics/particle/class_1/pid_accuracy": 0.9818770289421082, "metrics/particle/class_2/efficiency": 0.2978372275829315, "metrics/particle/class_2/pid_accuracy": 0.7268882989883423, "metrics/particle/class_3/efficiency": 0.6441035270690918, "metrics/particle/class_3/pid_accuracy": 0.9756197929382324, "metrics/particle/class_4/efficiency": 0.9082202911376953, "metrics/particle/class_4/pid_accuracy": 0.5158172249794006, "metrics/particle/class_5/efficiency": 0.9855700135231018, "metrics/particle/class_5/pid_accuracy": 0.69838947057724, "metrics/particle/count/bias_mean": -1.5565714836120605, "metrics/particle/count/mae": 4.665143013000488, "metrics/particle/count/prediction_mean": 90.59771728515625, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.046266525983810425, "metrics/particle/event/energy_response_mean": 1.0063228607177734, "metrics/particle/event/met_abs_error": 5.3312883377075195, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037024691700935364, "metrics/particle/event/scalar_pt_response_mean": 1.0102322101593018, "metrics/particle/event/vector_pt_closure": 0.03947809711098671, "metrics/particle/matched/delta_eta_abs_mean": 0.013180426321923733, "metrics/particle/matched/delta_phi_abs_mean": 0.013168545439839363, "metrics/particle/matched/delta_r_mean": 0.020818104967474937, "metrics/particle/matched/energy_relative_abs_mean": 0.09154995530843735, "metrics/particle/matched/pid_accuracy": 0.958248496055603, "metrics/particle/matched/pt_relative_abs_mean": 0.08587014675140381, "metrics/particle/matching/duplicate_fraction": 0.03482900932431221, "metrics/particle/matching/efficiency": 0.7246109247207642, "metrics/particle/matching/f1": 0.730782687664032, "metrics/particle/matching/purity": 0.737060546875, "metrics/particle/matching/target_coverage_dr0p05": 0.6622558236122131, "metrics/particle/matching/target_coverage_dr0p10": 0.7624728679656982, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6444472074508667, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8030632138252258}, "step_train_time": 16.215721607208252, "step_valid_time": 12.536348010995425, "step_total_time": 38826.75629210472, "elapsed_training_loop_seconds": 38826.75629210472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0107935944791462, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07089277713068964, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1424, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9582772543741588, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9602157788267026, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9592455372179185, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03978422117329737, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9522207267833109, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9541469993256911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9531828898619064, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9582772543741588, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0082558742738001, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0870035528339429, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1060, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9481216457960644, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9481216457960644, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9481216457960644, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0518783542039356, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9481216457960644, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0161950389798506, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08615234724107967, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 578, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9249201277955271, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9234449760765551, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07507987220447285, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9203821656050956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9233226837060703, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9218500797448166, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9219745222929936, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00421880337583, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05469078961473106, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1491, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1430, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.960455764075067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9610999329309189, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9607777405296681, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038900067069081154, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9584450402144772, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9590878604963112, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9587663426081127, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.960455764075067, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0071571196940174, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07007552908452142, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1047, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1045, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9423942394239424, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9492293744333635, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.94579945799458, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050770625566636446, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9405940594059405, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9474161378059837, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9439927732610659, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9423942394239424, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0089329341048148, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0738363824867263, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 634, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 587, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 585, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9229559748427673, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9258675078864353, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9244094488188976, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07413249211356467, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9198113207547169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9227129337539433, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9212598425196851, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9229559748427673, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9980250836857918, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13239590361624642, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3598, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2662, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2630, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7504933746828306, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7398554752640356, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7451364590622813, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.26014452473596444, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7414716661967861, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7309616453585325, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7361791462561231, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7504933746828306, "step_test_time": 36.44955790089443, "step_plot_time": 26.61264017492067, "step_validation_cycle_time": 76.30330561101437} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_310000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_310000.json new file mode 100644 index 0000000000000000000000000000000000000000..f86bfc6f2a03a9f1fcdfe29ce5875224c9e83415 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_310000.json @@ -0,0 +1 @@ +{"epoch": 0.7867720772051825, "train": {"Classification_binary": 2.3722052574157715, "Classification": 0.015000336803495884, "Regression_pt": 0.12769189476966858, "Regression_eta": 0.007996294647455215, "Regression_sin_phi": 0.007052815519273281, "Regression_cos_phi": 0.007433873135596514, "Regression_energy": 0.12681807577610016, "Total": 4.883780002593994, "Total_unweighted": 2.664198548067361}, "train_group": {"cld": {"Classification_binary": 2.322259363300072, "Classification": 0.016080524738944932, "Regression_pt": 0.13359374196051124, "Regression_eta": 0.014307683774320309, "Regression_sin_phi": 0.007305998395581891, "Regression_cos_phi": 0.007380488785234709, "Regression_energy": 0.1390751815752355, "Total": 5.026170752719491, "Total_unweighted": 2.640002982529901}, "clic": {"Classification_binary": 2.219496498952757, "Classification": 0.017720981724784, "Regression_pt": 0.12050869640287336, "Regression_eta": 0.010459312792257728, "Regression_sin_phi": 0.008897667335396682, "Regression_cos_phi": 0.008705393449674631, "Regression_energy": 0.12175484914500434, "Total": 4.716202896994513, "Total_unweighted": 2.5075433998027474}, "colliderml": {"Classification_binary": 2.281455020646791, "Classification": 0.02089965764668074, "Regression_pt": 0.2760796342884098, "Regression_eta": 0.058067423803312285, "Regression_sin_phi": 0.06494107017959203, "Regression_cos_phi": 0.06480866952522381, "Regression_energy": 0.31321857829351685, "Total": 9.238225298242885, "Total_unweighted": 3.0794700543835267}}, "valid": {"Classification_binary": 2.2830675992098723, "Classification": 0.017464282295920633, "Regression_pt": 0.15098882154984908, "Regression_eta": 0.018201160430908202, "Regression_sin_phi": 0.01568747650493275, "Regression_cos_phi": 0.015840541232715954, "Regression_energy": 0.15820818814364346, "Total": 5.528272594105114, "group/cld/Classification": 0.015918622625635027, "group/cld/Classification_binary": 2.302780313694731, "group/cld/Regression_cos_phi": 0.007401151860013921, "group/cld/Regression_energy": 0.1411718713476303, "group/cld/Regression_eta": 0.012392722545786107, "group/cld/Regression_pt": 0.13681515226972865, "group/cld/Regression_sin_phi": 0.006860802782342789, "group/cld/Total": 5.033486548890459, "group/cld/Total_unweighted": 2.6233406371258683, "group/clic/Classification": 0.017957301850014543, "group/clic/Classification_binary": 2.2663272289519614, "group/clic/Regression_cos_phi": 0.008280584786800629, "group/clic/Regression_energy": 0.12559271873311795, "group/clic/Regression_eta": 0.010888241707010473, "group/clic/Regression_pt": 0.12319554673864487, "group/clic/Regression_sin_phi": 0.00814354102662269, "group/clic/Total": 4.805992288792387, "group/clic/Total_unweighted": 2.560385163794173, "group/colliderml/Classification": 0.020556382834911346, "group/colliderml/Classification_binary": 2.2743353843688965, "group/colliderml/Regression_cos_phi": 0.06283863633871078, "group/colliderml/Regression_energy": 0.30406033992767334, "group/colliderml/Regression_eta": 0.05674516409635544, "group/colliderml/Regression_pt": 0.27426642179489136, "group/colliderml/Regression_sin_phi": 0.06377612054347992, "group/colliderml/Total": 9.1033935546875, "group/colliderml/Total_unweighted": 3.0565784499049187, "Total_unweighted": 2.659458069367842, "metrics/particle/class_1/efficiency": 0.9594258666038513, "metrics/particle/class_1/pid_accuracy": 0.9828810691833496, "metrics/particle/class_2/efficiency": 0.3003995418548584, "metrics/particle/class_2/pid_accuracy": 0.720399022102356, "metrics/particle/class_3/efficiency": 0.6636070013046265, "metrics/particle/class_3/pid_accuracy": 0.9761258363723755, "metrics/particle/class_4/efficiency": 0.9098164439201355, "metrics/particle/class_4/pid_accuracy": 0.4447368383407593, "metrics/particle/class_5/efficiency": 0.9711399674415588, "metrics/particle/class_5/pid_accuracy": 0.6775631308555603, "metrics/particle/count/bias_mean": -0.6994285583496094, "metrics/particle/count/mae": 4.5920000076293945, "metrics/particle/count/prediction_mean": 91.4548568725586, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.05021829530596733, "metrics/particle/event/energy_response_mean": 1.0169490575790405, "metrics/particle/event/met_abs_error": 5.649692058563232, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039523009210824966, "metrics/particle/event/scalar_pt_response_mean": 1.01631760597229, "metrics/particle/event/vector_pt_closure": 0.04032166302204132, "metrics/particle/matched/delta_eta_abs_mean": 0.013928801752626896, "metrics/particle/matched/delta_phi_abs_mean": 0.012328743003308773, "metrics/particle/matched/delta_r_mean": 0.020808232948184013, "metrics/particle/matched/energy_relative_abs_mean": 0.09257081151008606, "metrics/particle/matched/pid_accuracy": 0.9579037427902222, "metrics/particle/matched/pt_relative_abs_mean": 0.08456214517354965, "metrics/particle/matching/duplicate_fraction": 0.035208627581596375, "metrics/particle/matching/efficiency": 0.733701229095459, "metrics/particle/matching/f1": 0.7364961504936218, "metrics/particle/matching/purity": 0.739312469959259, "metrics/particle/matching/target_coverage_dr0p05": 0.6732126474380493, "metrics/particle/matching/target_coverage_dr0p10": 0.7685248255729675, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6569293737411499, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8113040328025818}, "step_train_time": 19.903067111968994, "step_valid_time": 12.670358126983047, "step_total_time": 40194.109524965286, "elapsed_training_loop_seconds": 40194.109524965286, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0074263143484972, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06994463180930921, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1419, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.959005376344086, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9596503026227303, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04099462365591398, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9549125168236877, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9536290322580645, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9542703429724277, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9602960969044414, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0102824766070893, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0843714695867559, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1121, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1062, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9543828264758497, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.951828724353256, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9531040643144261, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04817127564674398, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9499105545617174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9473684210526315, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9486377847253238, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9543828264758497, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0124719116601049, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09118606508084837, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 630, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 577, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 577, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9187898089171974, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9158730158730158, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9173290937996821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08412698412698413, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9187898089171974, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9158730158730158, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9173290937996821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9187898089171974, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0050502258879912, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05615182546320807, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1495, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1427, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9597855227882037, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9578595317725752, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9588215600937395, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042140468227424746, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9564343163538874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.954515050167224, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9554737194509542, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9597855227882037, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0084035539346652, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07319382516729878, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1096, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1050, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1047, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9450945094509451, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.958029197080292, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9515178975985501, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041970802919708027, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9423942394239424, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9552919708029197, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9487992750339828, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9450945094509451, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0106673422851205, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0732692801189967, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 638, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 588, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 585, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9245283018867925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9216300940438872, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9230769230769231, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07836990595611286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9198113207547169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9169278996865203, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9183673469387755, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9245283018867925, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038305688238216, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1281328871292985, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3584, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2689, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2663, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7581054412179307, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7502790178571429, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7541719253961576, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24972098214285715, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7507753030730194, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7430245535714286, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7468798205020334, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7581054412179307, "step_test_time": 36.73038741503842, "step_plot_time": 26.887930869939737, "step_validation_cycle_time": 77.01305914693512} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_320000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_320000.json new file mode 100644 index 0000000000000000000000000000000000000000..e059fc80e4e7f073cc7a4d37b8b8926cf1c25477 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_320000.json @@ -0,0 +1 @@ +{"epoch": 0.8121518216311562, "train": {"Classification_binary": 1.9387280941009521, "Classification": 0.017075276002287865, "Regression_pt": 0.09376001358032227, "Regression_eta": 0.01331102754920721, "Regression_sin_phi": 0.008541087619960308, "Regression_cos_phi": 0.006398370489478111, "Regression_energy": 0.09530312567949295, "Total": 3.9752817153930664, "Total_unweighted": 2.173116995021701}, "train_group": {"cld": {"Classification_binary": 2.3144406585006028, "Classification": 0.01619952460946711, "Regression_pt": 0.12910274624287546, "Regression_eta": 0.014561945653601631, "Regression_sin_phi": 0.007337450489608778, "Regression_cos_phi": 0.007584876126355237, "Regression_energy": 0.13482735288125258, "Total": 4.952262864098534, "Total_unweighted": 2.6240545545037635}, "clic": {"Classification_binary": 2.227942892177376, "Classification": 0.01767964772378167, "Regression_pt": 0.12133832594859385, "Regression_eta": 0.010413730267468772, "Regression_sin_phi": 0.008728204688899397, "Regression_cos_phi": 0.008693219979522292, "Regression_energy": 0.12223358880616948, "Total": 4.732907080364798, "Total_unweighted": 2.5170296095918117}, "colliderml": {"Classification_binary": 2.28336377544804, "Classification": 0.02080222812702169, "Regression_pt": 0.2723458567270646, "Regression_eta": 0.05821150766165407, "Regression_sin_phi": 0.06465357597973254, "Regression_cos_phi": 0.06481942211140741, "Regression_energy": 0.30914379535494624, "Total": 9.172857099825197, "Total_unweighted": 3.0733401614098668}}, "valid": {"Classification_binary": 2.2710356278852983, "Classification": 0.017455580017783426, "Regression_pt": 0.15001428777521306, "Regression_eta": 0.01816594817421653, "Regression_sin_phi": 0.015586508404124866, "Regression_cos_phi": 0.015800329771908848, "Regression_energy": 0.1574850082397461, "Total": 5.4998696067116475, "group/cld/Classification": 0.015950408387691416, "group/cld/Classification_binary": 2.2880083449343416, "group/cld/Regression_cos_phi": 0.007544762910680568, "group/cld/Regression_energy": 0.1381153553090197, "group/cld/Regression_eta": 0.012527560934107354, "group/cld/Regression_pt": 0.13353739393518327, "group/cld/Regression_sin_phi": 0.006882463997982918, "group/cld/Total": 4.9696385809715755, "group/cld/Total_unweighted": 2.6025662904090066, "group/clic/Classification": 0.01791786513430007, "group/clic/Classification_binary": 2.2524321535800365, "group/clic/Regression_cos_phi": 0.008178185909352403, "group/clic/Regression_energy": 0.12649477289078084, "group/clic/Regression_eta": 0.010775535664659865, "group/clic/Regression_pt": 0.1236779639061461, "group/clic/Regression_sin_phi": 0.007938099668381061, "group/clic/Total": 4.797806110787899, "group/clic/Total_unweighted": 2.5474145767536567, "group/colliderml/Classification": 0.02051909640431404, "group/colliderml/Classification_binary": 2.2758259773254395, "group/colliderml/Regression_cos_phi": 0.06244107708334923, "group/colliderml/Regression_energy": 0.3054172694683075, "group/colliderml/Regression_eta": 0.05643802508711815, "group/colliderml/Regression_pt": 0.275778204202652, "group/colliderml/Regression_sin_phi": 0.06362182646989822, "group/colliderml/Total": 9.119723320007324, "group/colliderml/Total_unweighted": 3.0600414760410786, "Total_unweighted": 2.6455432902682916, "metrics/particle/class_1/efficiency": 0.9643546938896179, "metrics/particle/class_1/pid_accuracy": 0.9842984080314636, "metrics/particle/class_2/efficiency": 0.29705551266670227, "metrics/particle/class_2/pid_accuracy": 0.7160818576812744, "metrics/particle/class_3/efficiency": 0.6718080043792725, "metrics/particle/class_3/pid_accuracy": 0.9788919687271118, "metrics/particle/class_4/efficiency": 0.9185953736305237, "metrics/particle/class_4/pid_accuracy": 0.44569939374923706, "metrics/particle/class_5/efficiency": 0.9826839566230774, "metrics/particle/class_5/pid_accuracy": 0.6989721059799194, "metrics/particle/count/bias_mean": 0.20114286243915558, "metrics/particle/count/mae": 4.098285675048828, "metrics/particle/count/prediction_mean": 92.35543060302734, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.045581888407468796, "metrics/particle/event/energy_response_mean": 0.9921732544898987, "metrics/particle/event/met_abs_error": 5.013549327850342, "metrics/particle/event/scalar_pt_relative_abs_error": 0.035472605377435684, "metrics/particle/event/scalar_pt_response_mean": 0.9968744516372681, "metrics/particle/event/vector_pt_closure": 0.03849130868911743, "metrics/particle/matched/delta_eta_abs_mean": 0.012232049368321896, "metrics/particle/matched/delta_phi_abs_mean": 0.012701587751507759, "metrics/particle/matched/delta_r_mean": 0.01974133588373661, "metrics/particle/matched/energy_relative_abs_mean": 0.08813870698213577, "metrics/particle/matched/pid_accuracy": 0.9598898887634277, "metrics/particle/matched/pt_relative_abs_mean": 0.08275514841079712, "metrics/particle/matching/duplicate_fraction": 0.03282969072461128, "metrics/particle/matching/efficiency": 0.7389594912528992, "metrics/particle/matching/f1": 0.7381539344787598, "metrics/particle/matching/purity": 0.7373501062393188, "metrics/particle/matching/target_coverage_dr0p05": 0.6809449791908264, "metrics/particle/matching/target_coverage_dr0p10": 0.7714887857437134, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6656600832939148, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8183419108390808}, "step_train_time": 14.224489450454712, "step_valid_time": 12.75100814900361, "step_total_time": 41550.293258190155, "elapsed_training_loop_seconds": 41550.293258190155, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9945975826564126, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06488838187662327, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.959005376344086, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9596503026227303, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04099462365591398, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9549731182795699, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9556153328850033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9602960969044414, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9941815346736855, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07947855570042495, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1076, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1071, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.962432915921288, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9581478183437222, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9602855867916109, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041852181656277826, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9579606440071556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9536954585930543, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9558232931726908, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.962432915921288, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9957320728661526, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07781747702355357, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 621, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 575, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 574, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9156050955414012, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9259259259259259, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9207365892714171, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07407407407407407, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9140127388535032, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9243156199677939, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9191353082465973, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9156050955414012, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.994492529852041, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05366838695051468, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1496, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1430, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.960455764075067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9578877005347594, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9591700133868809, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042112299465240643, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9584450402144772, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9558823529411765, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9571619812583668, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.960455764075067, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9986183430269927, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06672473551961045, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1101, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1043, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1039, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9387938793879388, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9473206176203451, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9430379746835443, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05267938237965486, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9351935193519352, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9436875567665758, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9394213381555153, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9387938793879388, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9986735950109837, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.061177013472260566, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 637, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 589, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 586, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9261006289308176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9246467817896389, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9253731343283582, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07535321821036106, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9213836477987422, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9199372056514914, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9206598586017282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9261006289308176, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9883708250280614, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1268284160060138, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3576, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2701, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2666, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7614885819001973, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7553131991051454, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7583883195282887, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2446868008948546, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7516210882435861, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7455257270693513, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7485609995788292, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7614885819001973, "step_test_time": 36.7599833169952, "step_plot_time": 26.56728286901489, "step_validation_cycle_time": 76.79119192296639} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_330000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_330000.json new file mode 100644 index 0000000000000000000000000000000000000000..7d1168fecbcd5101d6c8d4590e802183c715a072 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_330000.json @@ -0,0 +1 @@ +{"epoch": 0.8375315660571298, "train": {"Classification_binary": 2.3130109310150146, "Classification": 0.02029149793088436, "Regression_pt": 0.33122718334198, "Regression_eta": 0.0648571252822876, "Regression_sin_phi": 0.06353744119405746, "Regression_cos_phi": 0.0639638900756836, "Regression_energy": 0.3965759575366974, "Total": 10.361837387084961, "Total_unweighted": 3.253464026376605}, "train_group": {"cld": {"Classification_binary": 2.298433489627666, "Classification": 0.016108273578858055, "Regression_pt": 0.1304934248328209, "Regression_eta": 0.01251941436884908, "Regression_sin_phi": 0.007519831410371461, "Regression_cos_phi": 0.007520243575846827, "Regression_energy": 0.13557595468766698, "Total": 4.94308421633265, "Total_unweighted": 2.6081706320820794}, "clic": {"Classification_binary": 2.2133681928073323, "Classification": 0.01746378173665689, "Regression_pt": 0.11905808641656383, "Regression_eta": 0.010182482720417035, "Regression_sin_phi": 0.008684253491399748, "Regression_cos_phi": 0.008567541925091151, "Regression_energy": 0.11986766414375634, "Total": 4.674094887467118, "Total_unweighted": 2.497192003241217}, "colliderml": {"Classification_binary": 2.2736965710531454, "Classification": 0.020647838444053056, "Regression_pt": 0.2722031937209432, "Regression_eta": 0.057820466876833024, "Regression_sin_phi": 0.06446399926737754, "Regression_cos_phi": 0.06417644226890125, "Regression_energy": 0.3081424443307751, "Total": 9.140900943093671, "Total_unweighted": 3.0611509559620282}}, "valid": {"Classification_binary": 2.2541175148703836, "Classification": 0.01743599826639349, "Regression_pt": 0.1489242900501598, "Regression_eta": 0.01815257505937056, "Regression_sin_phi": 0.015561679276553067, "Regression_cos_phi": 0.0157604455947876, "Regression_energy": 0.15641727447509765, "Total": 5.464110218394886, "group/cld/Classification": 0.015836420211386173, "group/cld/Classification_binary": 2.268495924929355, "group/cld/Regression_cos_phi": 0.00728715163596133, "group/cld/Regression_energy": 0.1365792193311326, "group/cld/Regression_eta": 0.012679046772895976, "group/cld/Regression_pt": 0.13191643167049327, "group/cld/Regression_sin_phi": 0.006722291099264267, "group/cld/Total": 4.918439824530419, "group/cld/Total_unweighted": 2.5795164856504886, "group/clic/Classification": 0.01792961612660834, "group/clic/Classification_binary": 2.2352288428773273, "group/clic/Regression_cos_phi": 0.008296226567410409, "group/clic/Regression_energy": 0.12631407189876476, "group/clic/Regression_eta": 0.010552116531006833, "group/clic/Regression_pt": 0.12373239436048142, "group/clic/Regression_sin_phi": 0.008100020758649136, "group/clic/Total": 4.782196044921875, "group/clic/Total_unweighted": 2.530153289120248, "group/colliderml/Classification": 0.020684747025370598, "group/colliderml/Classification_binary": 2.267364740371704, "group/colliderml/Regression_cos_phi": 0.06257690489292145, "group/colliderml/Regression_energy": 0.3031195402145386, "group/colliderml/Regression_eta": 0.05655742064118385, "group/colliderml/Regression_pt": 0.272885799407959, "group/colliderml/Regression_sin_phi": 0.06344600766897202, "group/colliderml/Total": 9.07014274597168, "group/colliderml/Total_unweighted": 3.0466351602226496, "Total_unweighted": 2.6263697775927457, "metrics/particle/class_1/efficiency": 0.9625778198242188, "metrics/particle/class_1/pid_accuracy": 0.9833384156227112, "metrics/particle/class_2/efficiency": 0.29084512591362, "metrics/particle/class_2/pid_accuracy": 0.7295804023742676, "metrics/particle/class_3/efficiency": 0.6486301422119141, "metrics/particle/class_3/pid_accuracy": 0.9770177006721497, "metrics/particle/class_4/efficiency": 0.9130087494850159, "metrics/particle/class_4/pid_accuracy": 0.4720279574394226, "metrics/particle/class_5/efficiency": 0.972582995891571, "metrics/particle/class_5/pid_accuracy": 0.718100905418396, "metrics/particle/count/bias_mean": -2.1222856044769287, "metrics/particle/count/mae": 5.0034284591674805, "metrics/particle/count/prediction_mean": 90.03199768066406, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.047249019145965576, "metrics/particle/event/energy_response_mean": 1.0115244388580322, "metrics/particle/event/met_abs_error": 5.580324649810791, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03757334500551224, "metrics/particle/event/scalar_pt_response_mean": 1.0141041278839111, "metrics/particle/event/vector_pt_closure": 0.040322963148355484, "metrics/particle/matched/delta_eta_abs_mean": 0.011956988833844662, "metrics/particle/matched/delta_phi_abs_mean": 0.012710286304354668, "metrics/particle/matched/delta_r_mean": 0.01949244551360607, "metrics/particle/matched/energy_relative_abs_mean": 0.09006183594465256, "metrics/particle/matched/pid_accuracy": 0.9598263502120972, "metrics/particle/matched/pt_relative_abs_mean": 0.08392374962568283, "metrics/particle/matching/duplicate_fraction": 0.03323262929916382, "metrics/particle/matching/efficiency": 0.7269858121871948, "metrics/particle/matching/f1": 0.7354544401168823, "metrics/particle/matching/purity": 0.7441227436065674, "metrics/particle/matching/target_coverage_dr0p05": 0.6754510998725891, "metrics/particle/matching/target_coverage_dr0p10": 0.7636572122573853, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6526446342468262, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8040739297866821}, "step_train_time": 14.182647228240967, "step_valid_time": 12.93136778601911, "step_total_time": 42876.80804038048, "elapsed_training_loop_seconds": 42876.80804038048, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0087145400278423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06604898753664457, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9636608344549125, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9623655913978495, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9630127774041695, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03763440860215054, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9576043068640646, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9563172043010753, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.956960322797579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9636608344549125, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0100583551648254, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07872266593526811, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1121, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1072, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1068, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9588550983899821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9562890276538805, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9575703439035284, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043710972346119537, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552772808586762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9527207850133809, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9539973202322466, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9588550983899821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0131760996813448, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08589268485807633, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 576, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 573, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9171974522292994, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9171974522292994, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9171974522292994, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08280254777070063, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9124203821656051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9124203821656051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9124203821656051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9171974522292994, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0066042134569906, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05210104247320757, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1494, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1428, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1424, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9571045576407506, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9558232931726908, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.956463496316142, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04417670682730924, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9544235924932976, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9531459170013387, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9537843268586738, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9571045576407506, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0087399090722438, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07067254247262844, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1104, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1046, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1043, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9414941494149415, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.947463768115942, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.944469525959368, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05253623188405797, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9387938793879388, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9447463768115942, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9417607223476298, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9414941494149415, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.011419055502893, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06941816834634817, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 640, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 591, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 590, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9234375, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9263322884012539, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0765625, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.921875, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9247648902821317, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9292452830188679, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020281687822787, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12864434424654103, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3569, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2669, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2637, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7524668734141529, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7478285233959092, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7501405283867342, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2521714766040908, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7434451649281083, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.738862426449986, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7411467116357504, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7524668734141529, "step_test_time": 36.77640465507284, "step_plot_time": 26.807219136040658, "step_validation_cycle_time": 77.22450633405242} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_340000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_340000.json new file mode 100644 index 0000000000000000000000000000000000000000..bb3829ffd85e86b72c973ee768d90d09f7315161 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_340000.json @@ -0,0 +1 @@ +{"epoch": 0.8629113104831034, "train": {"Classification_binary": 2.090768337249756, "Classification": 0.014944297261536121, "Regression_pt": 0.09118157625198364, "Regression_eta": 0.02552129328250885, "Regression_sin_phi": 0.004827780183404684, "Regression_cos_phi": 0.00915827788412571, "Regression_energy": 0.09526071697473526, "Total": 4.1318206787109375, "Total_unweighted": 2.33166227908805}, "train_group": {"cld": {"Classification_binary": 2.2702378598515858, "Classification": 0.015742413150313907, "Regression_pt": 0.1276709948426592, "Regression_eta": 0.013454136780245665, "Regression_sin_phi": 0.007098522189234991, "Regression_cos_phi": 0.007413204836040111, "Regression_energy": 0.13250086817912712, "Total": 4.861097442175814, "Total_unweighted": 2.5741179998292067}, "clic": {"Classification_binary": 2.1909759778518216, "Classification": 0.017475108551728475, "Regression_pt": 0.11893097018157398, "Regression_eta": 0.010259762200087801, "Regression_sin_phi": 0.008738077147066727, "Regression_cos_phi": 0.008784934353606368, "Regression_energy": 0.11992464961858841, "Total": 4.654670680249417, "Total_unweighted": 2.4750894799044736}, "colliderml": {"Classification_binary": 2.2663357852099537, "Classification": 0.020676896216588334, "Regression_pt": 0.27452183015890663, "Regression_eta": 0.058179729044795396, "Regression_sin_phi": 0.06475318681907367, "Regression_cos_phi": 0.0643638290643871, "Regression_energy": 0.3111616661688229, "Total": 9.184380754694208, "Total_unweighted": 3.0599929226825275}}, "valid": {"Classification_binary": 2.249590925736861, "Classification": 0.017246701500632546, "Regression_pt": 0.1475383238358931, "Regression_eta": 0.01808634562925859, "Regression_sin_phi": 0.015521014820445668, "Regression_cos_phi": 0.015626480362632058, "Regression_energy": 0.15510846918279475, "Total": 5.4323336514559655, "group/cld/Classification": 0.015627209176408485, "group/cld/Classification_binary": 2.2695943954143116, "group/cld/Regression_cos_phi": 0.0073387451628421215, "group/cld/Regression_energy": 0.13760884264682202, "group/cld/Regression_eta": 0.012503304380051632, "group/cld/Regression_pt": 0.1328511745371717, "group/cld/Regression_sin_phi": 0.0068992858237408574, "group/cld/Total": 4.934394024788065, "group/cld/Total_unweighted": 2.582422957141348, "group/clic/Classification": 0.01780154730411286, "group/clic/Classification_binary": 2.2280352977996176, "group/clic/Regression_cos_phi": 0.007964764503722495, "group/clic/Regression_energy": 0.12431945192052964, "group/clic/Regression_eta": 0.010673547044713447, "group/clic/Regression_pt": 0.12156484482136179, "group/clic/Regression_sin_phi": 0.007925346176675025, "group/clic/Total": 4.733509794194648, "group/clic/Total_unweighted": 2.5182847995707327, "group/colliderml/Classification": 0.02037409506738186, "group/colliderml/Classification_binary": 2.254152536392212, "group/colliderml/Regression_cos_phi": 0.062477994710206985, "group/colliderml/Regression_energy": 0.2969563901424408, "group/colliderml/Regression_eta": 0.05626162141561508, "group/colliderml/Regression_pt": 0.2669789493083954, "group/colliderml/Regression_sin_phi": 0.06315960735082626, "group/colliderml/Total": 8.947834014892578, "group/colliderml/Total_unweighted": 3.0203611943870783, "Total_unweighted": 2.6187182610685174, "metrics/particle/class_1/efficiency": 0.9597194194793701, "metrics/particle/class_1/pid_accuracy": 0.9830955862998962, "metrics/particle/class_2/efficiency": 0.3000086843967438, "metrics/particle/class_2/pid_accuracy": 0.7190214395523071, "metrics/particle/class_3/efficiency": 0.6631040573120117, "metrics/particle/class_3/pid_accuracy": 0.9782567620277405, "metrics/particle/class_4/efficiency": 0.8970471024513245, "metrics/particle/class_4/pid_accuracy": 0.467971533536911, "metrics/particle/class_5/efficiency": 0.9769119620323181, "metrics/particle/class_5/pid_accuracy": 0.7223042845726013, "metrics/particle/count/bias_mean": -1.173142910003662, "metrics/particle/count/mae": 4.485142707824707, "metrics/particle/count/prediction_mean": 90.98114013671875, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.047012001276016235, "metrics/particle/event/energy_response_mean": 1.0051183700561523, "metrics/particle/event/met_abs_error": 5.409593105316162, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03715520352125168, "metrics/particle/event/scalar_pt_response_mean": 1.010980248451233, "metrics/particle/event/vector_pt_closure": 0.03898704797029495, "metrics/particle/matched/delta_eta_abs_mean": 0.011882476508617401, "metrics/particle/matched/delta_phi_abs_mean": 0.01221716869622469, "metrics/particle/matched/delta_r_mean": 0.01908780075609684, "metrics/particle/matched/energy_relative_abs_mean": 0.08994337171316147, "metrics/particle/matched/pid_accuracy": 0.9593443274497986, "metrics/particle/matched/pt_relative_abs_mean": 0.08502358198165894, "metrics/particle/matching/duplicate_fraction": 0.035448476672172546, "metrics/particle/matching/efficiency": 0.7334656119346619, "metrics/particle/matching/f1": 0.7381641268730164, "metrics/particle/matching/purity": 0.7429232001304626, "metrics/particle/matching/target_coverage_dr0p05": 0.6865071058273315, "metrics/particle/matching/target_coverage_dr0p10": 0.7693867683410645, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6535375714302063, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8103243112564087}, "step_train_time": 14.831167936325073, "step_valid_time": 12.658834581030533, "step_total_time": 44215.36531043053, "elapsed_training_loop_seconds": 44215.36531043053, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0101073716833082, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06816883237906801, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1489, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1420, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9597044996642041, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9606722689075631, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.040295500335795834, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.955585464333782, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9536601746138348, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9546218487394958, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9616419919246298, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0098367981186762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07866097393654892, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1120, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1074, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.960644007155635, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9589285714285715, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9597855227882037, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04107142857142857, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9543828264758497, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9526785714285714, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9535299374441466, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.960644007155635, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0154861348178574, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08429277402852187, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 623, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 578, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 575, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9203821656050956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.92776886035313, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9240607513988809, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07223113964686999, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9156050955414012, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9229534510433387, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9192645883293366, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9203821656050956, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042533321657368, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.053417873381763736, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.960455764075067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.960455764075067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.960455764075067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.039544235924932974, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.957774798927614, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.957774798927614, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.957774798927614, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.960455764075067, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0006891373751203, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06809010821133121, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1099, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1046, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1044, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9414941494149415, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9517743403093721, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9466063348416289, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048225659690627844, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9396939693969397, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9499545040946314, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9447963800904977, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9414941494149415, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0066648237620865, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07089091634280409, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 638, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 591, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 589, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9263322884012539, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9277864992150706, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07366771159874608, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9261006289308176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9231974921630094, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9246467817896389, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9292452830188679, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011075433523149, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13062207802518633, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3577, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2699, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2678, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7609247251198196, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7545429130556333, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7577203818079731, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2454570869443668, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7550042289258528, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7486720715683534, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7518248175182481, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7609247251198196, "step_test_time": 36.855214686947875, "step_plot_time": 26.835964649915695, "step_validation_cycle_time": 77.08047406293917} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_350000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_350000.json new file mode 100644 index 0000000000000000000000000000000000000000..92f3a7d790de1079f06160aa5b951380a788a98d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_350000.json @@ -0,0 +1 @@ +{"epoch": 0.8882910549090771, "train": {"Classification_binary": 2.2115228176116943, "Classification": 0.021936001256108284, "Regression_pt": 0.13886983692646027, "Regression_eta": 0.010375022888183594, "Regression_sin_phi": 0.013589621521532536, "Regression_cos_phi": 0.0052430094219744205, "Regression_energy": 0.13460773229599, "Total": 5.046122074127197, "Total_unweighted": 2.5361440419219434}, "train_group": {"cld": {"Classification_binary": 2.2661206657822066, "Classification": 0.01580112826851037, "Regression_pt": 0.12676421179248765, "Regression_eta": 0.01316244148106703, "Regression_sin_phi": 0.007318176512323969, "Regression_cos_phi": 0.00736799045163836, "Regression_energy": 0.13211269904901315, "Total": 4.846737427754445, "Total_unweighted": 2.568647313337247}, "clic": {"Classification_binary": 2.1692020239944227, "Classification": 0.017021294538221675, "Regression_pt": 0.11806560381354686, "Regression_eta": 0.010504985671535044, "Regression_sin_phi": 0.008954409307041791, "Regression_cos_phi": 0.008653432523976722, "Regression_energy": 0.11906130188061091, "Total": 4.615120138236862, "Total_unweighted": 2.451463051729356}, "colliderml": {"Classification_binary": 2.2714474172563524, "Classification": 0.020718613287081588, "Regression_pt": 0.26953293669510175, "Regression_eta": 0.058356761820502465, "Regression_sin_phi": 0.06463287384794639, "Regression_cos_phi": 0.06480567104689351, "Regression_energy": 0.3068525386196715, "Total": 9.117551802156925, "Total_unweighted": 3.05634681257355}}, "valid": {"Classification_binary": 2.235052628950639, "Classification": 0.017057518525557085, "Regression_pt": 0.1458004344593395, "Regression_eta": 0.017890100045637652, "Regression_sin_phi": 0.015512914007360285, "Regression_cos_phi": 0.015598514947024258, "Regression_energy": 0.1533634532581676, "Total": 5.385125732421875, "group/cld/Classification": 0.015396478328299015, "group/cld/Classification_binary": 2.247240269437749, "group/cld/Regression_cos_phi": 0.007292453912978477, "group/cld/Regression_energy": 0.1339826076588732, "group/cld/Regression_eta": 0.012263748239963613, "group/cld/Regression_pt": 0.12940843054588805, "group/cld/Regression_sin_phi": 0.006789173851621912, "group/cld/Total": 4.847997462495845, "group/cld/Total_unweighted": 2.5523731619753733, "group/clic/Classification": 0.017604602144119588, "group/clic/Classification_binary": 2.2147465158016124, "group/clic/Regression_cos_phi": 0.007966781550265372, "group/clic/Regression_energy": 0.1229458565407611, "group/clic/Regression_eta": 0.010444116085133653, "group/clic/Regression_pt": 0.12023694464500914, "group/clic/Regression_sin_phi": 0.00801074631670688, "group/clic/Total": 4.695220947265625, "group/clic/Total_unweighted": 2.501955563083608, "group/colliderml/Classification": 0.020329756662249565, "group/colliderml/Classification_binary": 2.258902072906494, "group/colliderml/Regression_cos_phi": 0.06241580471396446, "group/colliderml/Regression_energy": 0.29964664578437805, "group/colliderml/Regression_eta": 0.05629010498523712, "group/colliderml/Regression_pt": 0.26904430985450745, "group/colliderml/Regression_sin_phi": 0.06317652761936188, "group/colliderml/Total": 8.98953914642334, "group/colliderml/Total_unweighted": 3.0298052225261927, "Total_unweighted": 2.6002755641937254, "metrics/particle/class_1/efficiency": 0.9630877375602722, "metrics/particle/class_1/pid_accuracy": 0.984117329120636, "metrics/particle/class_2/efficiency": 0.29579606652259827, "metrics/particle/class_2/pid_accuracy": 0.7223609089851379, "metrics/particle/class_3/efficiency": 0.6715285778045654, "metrics/particle/class_3/pid_accuracy": 0.9790704250335693, "metrics/particle/class_4/efficiency": 0.9201915264129639, "metrics/particle/class_4/pid_accuracy": 0.4787510931491852, "metrics/particle/class_5/efficiency": 0.98124098777771, "metrics/particle/class_5/pid_accuracy": 0.6926470398902893, "metrics/particle/count/bias_mean": -0.8994285464286804, "metrics/particle/count/mae": 4.260571479797363, "metrics/particle/count/prediction_mean": 91.2548599243164, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.043943293392658234, "metrics/particle/event/energy_response_mean": 0.9979175329208374, "metrics/particle/event/met_abs_error": 4.962460994720459, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03435813635587692, "metrics/particle/event/scalar_pt_response_mean": 0.9997390508651733, "metrics/particle/event/vector_pt_closure": 0.03727809339761734, "metrics/particle/matched/delta_eta_abs_mean": 0.011590716429054737, "metrics/particle/matched/delta_phi_abs_mean": 0.012830332852900028, "metrics/particle/matched/delta_r_mean": 0.019326671957969666, "metrics/particle/matched/energy_relative_abs_mean": 0.0875464528799057, "metrics/particle/matched/pid_accuracy": 0.9605433344841003, "metrics/particle/matched/pt_relative_abs_mean": 0.08219370245933533, "metrics/particle/matching/duplicate_fraction": 0.03114667907357216, "metrics/particle/matching/efficiency": 0.7381533980369568, "metrics/particle/matching/f1": 0.741773247718811, "metrics/particle/matching/purity": 0.7454288005828857, "metrics/particle/matching/target_coverage_dr0p05": 0.6844918727874756, "metrics/particle/matching/target_coverage_dr0p10": 0.7702176570892334, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6640850901603699, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8159980177879333}, "step_train_time": 17.16762089729309, "step_valid_time": 12.617169499048032, "step_total_time": 45577.6702914238, "elapsed_training_loop_seconds": 45577.6702914238, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018626971327964, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06677598915122407, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1424, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9582772543741588, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9595687331536388, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.958922558922559, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04043126684636118, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9535666218034994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9548517520215634, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9542087542087542, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9582772543741588, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9990102391344331, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07358262881764632, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1068, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1066, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552772808586762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9552772808586762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9552772808586762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044722719141323794, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552772808586762, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0039996526994535, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07600157722155787, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 574, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 574, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9140127388535032, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9198717948717948, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9169329073482428, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08012820512820513, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9140127388535032, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9198717948717948, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9169329073482428, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9140127388535032, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9987010886312633, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.049125325486468974, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1494, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1438, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1435, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9638069705093834, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9625167336010709, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9631614199598124, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03748326639892905, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9617962466487936, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9605087014725568, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9611520428667113, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9638069705093834, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9993676321946139, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06449860257735929, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1098, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1047, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1044, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9423942394239424, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.953551912568306, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9479402444545043, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04644808743169399, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9396939693969397, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9508196721311475, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9452240832956089, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9423942394239424, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003475979169792, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06266999094518066, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 593, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 592, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06761006289308176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9308176100628931, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9308176100628931, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9308176100628931, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9323899371069182, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9943203637936217, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12328328939389122, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3568, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2697, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2664, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7603608683394418, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7558856502242153, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7581166549543219, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24411434977578475, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7510572314632084, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7466367713004485, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7488404778636683, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7603608683394418, "step_test_time": 36.87693386001047, "step_plot_time": 26.69066948897671, "step_validation_cycle_time": 76.94335634808522} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_360000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_360000.json new file mode 100644 index 0000000000000000000000000000000000000000..5ed6d199d041c40ca97b3bdf75f616140738102c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_360000.json @@ -0,0 +1 @@ +{"epoch": 0.9136707993350507, "train": {"Classification_binary": 2.2781434059143066, "Classification": 0.021077806130051613, "Regression_pt": 0.2704195976257324, "Regression_eta": 0.05777047574520111, "Regression_sin_phi": 0.06489641219377518, "Regression_cos_phi": 0.06924661993980408, "Regression_energy": 0.29304587841033936, "Total": 9.1003999710083, "Total_unweighted": 3.0546001959592104}, "train_group": {"cld": {"Classification_binary": 2.246987384002846, "Classification": 0.01586675512193291, "Regression_pt": 0.12466374422247345, "Regression_eta": 0.014773149408998567, "Regression_sin_phi": 0.007172657243974574, "Regression_cos_phi": 0.007152844090000712, "Regression_energy": 0.12949929724226486, "Total": 4.7958561479150355, "Total_unweighted": 2.546115831332491}, "clic": {"Classification_binary": 2.161717756374462, "Classification": 0.017174653093646595, "Regression_pt": 0.11672800953860756, "Regression_eta": 0.010060774644853564, "Regression_sin_phi": 0.008378686257512183, "Regression_cos_phi": 0.008279803675367264, "Regression_energy": 0.11758628277270286, "Total": 4.572785555779397, "Total_unweighted": 2.439925966357152}, "colliderml": {"Classification_binary": 2.258878083999999, "Classification": 0.020498912965510777, "Regression_pt": 0.271521285667034, "Regression_eta": 0.05784817870907084, "Regression_sin_phi": 0.06420048156719722, "Regression_cos_phi": 0.0646887166749039, "Regression_energy": 0.30867432860914107, "Total": 9.123825698555587, "Total_unweighted": 3.046309988192857}}, "valid": {"Classification_binary": 2.2381222811612216, "Classification": 0.01711880943991921, "Regression_pt": 0.14543975483287463, "Regression_eta": 0.01806464737111872, "Regression_sin_phi": 0.015471158244393088, "Regression_cos_phi": 0.01570317961952903, "Regression_energy": 0.15297168384898793, "Total": 5.384668523615057, "group/cld/Classification": 0.015640772403554715, "group/cld/Classification_binary": 2.2495401260700634, "group/cld/Regression_cos_phi": 0.007398985801859105, "group/cld/Regression_energy": 0.13210151550617624, "group/cld/Regression_eta": 0.012415546051999356, "group/cld/Regression_pt": 0.1274942641562604, "group/cld/Regression_sin_phi": 0.006774987946165369, "group/cld/Total": 4.824436268907912, "group/cld/Total_unweighted": 2.551366197936079, "group/clic/Classification": 0.017535233751256415, "group/clic/Classification_binary": 2.2187051975980716, "group/clic/Regression_cos_phi": 0.008112500956717958, "group/clic/Regression_energy": 0.12417815593962973, "group/clic/Regression_eta": 0.01066355121896622, "group/clic/Regression_pt": 0.12134031539267683, "group/clic/Regression_sin_phi": 0.007922669674487823, "group/clic/Total": 4.719104523354388, "group/clic/Total_unweighted": 2.508457624531806, "group/colliderml/Classification": 0.020237306132912636, "group/colliderml/Classification_binary": 2.261618137359619, "group/colliderml/Regression_cos_phi": 0.06239437684416771, "group/colliderml/Regression_energy": 0.29885876178741455, "group/colliderml/Regression_eta": 0.056399594992399216, "group/colliderml/Regression_pt": 0.2689465582370758, "group/colliderml/Regression_sin_phi": 0.06318987905979156, "group/colliderml/Total": 8.985455513000488, "group/colliderml/Total_unweighted": 3.0316446144133806, "Total_unweighted": 2.602891514518044, "metrics/particle/class_1/efficiency": 0.9627014398574829, "metrics/particle/class_1/pid_accuracy": 0.9823775887489319, "metrics/particle/class_2/efficiency": 0.299791544675827, "metrics/particle/class_2/pid_accuracy": 0.7196871042251587, "metrics/particle/class_3/efficiency": 0.6692373156547546, "metrics/particle/class_3/pid_accuracy": 0.9782055020332336, "metrics/particle/class_4/efficiency": 0.9010375142097473, "metrics/particle/class_4/pid_accuracy": 0.5349867343902588, "metrics/particle/class_5/efficiency": 0.9797979593276978, "metrics/particle/class_5/pid_accuracy": 0.7334315180778503, "metrics/particle/count/bias_mean": -0.41428571939468384, "metrics/particle/count/mae": 4.192571640014648, "metrics/particle/count/prediction_mean": 91.73999786376953, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.047696731984615326, "metrics/particle/event/energy_response_mean": 1.0125561952590942, "metrics/particle/event/met_abs_error": 5.449781894683838, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037336163222789764, "metrics/particle/event/scalar_pt_response_mean": 1.0130513906478882, "metrics/particle/event/vector_pt_closure": 0.03888469561934471, "metrics/particle/matched/delta_eta_abs_mean": 0.011516932398080826, "metrics/particle/matched/delta_phi_abs_mean": 0.012453536503016949, "metrics/particle/matched/delta_r_mean": 0.018974602222442627, "metrics/particle/matched/energy_relative_abs_mean": 0.08834020048379898, "metrics/particle/matched/pid_accuracy": 0.959779679775238, "metrics/particle/matched/pt_relative_abs_mean": 0.083429716527462, "metrics/particle/matching/duplicate_fraction": 0.034744150936603546, "metrics/particle/matching/efficiency": 0.7373968958854675, "metrics/particle/matching/f1": 0.7390581369400024, "metrics/particle/matching/purity": 0.740726888179779, "metrics/particle/matching/target_coverage_dr0p05": 0.6887021660804749, "metrics/particle/matching/target_coverage_dr0p10": 0.7725491523742676, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6606436371803284, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.814329981803894}, "step_train_time": 13.926343441009521, "step_valid_time": 13.34908415703103, "step_total_time": 46929.81609082222, "elapsed_training_loop_seconds": 46929.81609082222, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0084359013668607, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06655282662502271, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1491, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9610999329309189, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.962714141753443, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038900067069081154, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9570757880617036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9586832381592207, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9643337819650067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0063508561026826, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07981850397063928, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1065, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1058, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9525939177101967, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9483526268922529, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9504685408299867, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.051647373107747106, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9463327370304114, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9421193232413179, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9442213297634985, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9525939177101967, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0108610147830328, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08517345018303346, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 580, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 577, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9235668789808917, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9294871794871795, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9265175718849841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07051282051282051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9187898089171974, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9246794871794872, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9217252396166135, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9235668789808917, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043042305680134, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05372702423498499, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1498, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1442, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1439, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9664879356568364, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9626168224299065, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9645484949832775, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.037383177570093455, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9644772117962467, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9606141522029372, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9625418060200669, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9664879356568364, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058082251318219, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06821137782968256, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1106, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1055, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1053, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9495949594959496, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.953887884267631, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9517365809652684, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0461121157323689, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9477947794779478, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9520795660036167, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9499323410013532, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9495949594959496, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.009277826644086, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0680958648442852, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 641, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 590, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 588, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9204368174726989, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9240407204385278, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07956318252730109, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9245283018867925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9173166926677067, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9209083790133125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9276729559748428, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043537308686386, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12838458101391803, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3604, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2693, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2661, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7592331547786862, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7472253052164262, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7531813732345126, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2527746947835738, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7502114462926417, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.73834628190899, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7442315760033562, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7592331547786862, "step_test_time": 37.18565339804627, "step_plot_time": 26.695320133003406, "step_validation_cycle_time": 77.95250170095824} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_370000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_370000.json new file mode 100644 index 0000000000000000000000000000000000000000..bc9525f5a1a35c14349504b2ec0a9042cbdaeeaa --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_370000.json @@ -0,0 +1 @@ +{"epoch": 0.9390505437610244, "train": {"Classification_binary": 2.0555708408355713, "Classification": 0.012696812860667706, "Regression_pt": 0.12534017860889435, "Regression_eta": 0.028318464756011963, "Regression_sin_phi": 0.009651659987866879, "Regression_cos_phi": 0.008437680080533028, "Regression_energy": 0.13907960057258606, "Total": 4.755976676940918, "Total_unweighted": 2.3790952377021313}, "train_group": {"cld": {"Classification_binary": 2.2450783420465665, "Classification": 0.015690889914971803, "Regression_pt": 0.1269704047032816, "Regression_eta": 0.013632322576532732, "Regression_sin_phi": 0.0070727754211399015, "Regression_cos_phi": 0.007195422380554774, "Regression_energy": 0.13200360137277733, "Total": 4.823229739052093, "Total_unweighted": 2.5476437584158247}, "clic": {"Classification_binary": 2.1679013849378705, "Classification": 0.01698823090553194, "Regression_pt": 0.11760485176746552, "Regression_eta": 0.01013550914015744, "Regression_sin_phi": 0.00836020133978276, "Regression_cos_phi": 0.00850204107779506, "Regression_energy": 0.1181836293803321, "Total": 4.591846991587688, "Total_unweighted": 2.4476758485489354}, "colliderml": {"Classification_binary": 2.251460251507458, "Classification": 0.020447288178377324, "Regression_pt": 0.2669094317668193, "Regression_eta": 0.05791083477191381, "Regression_sin_phi": 0.0644096997272861, "Regression_cos_phi": 0.06410146504640579, "Regression_energy": 0.30372496982952496, "Total": 9.033889783395303, "Total_unweighted": 3.0289639408277855}}, "valid": {"Classification_binary": 2.2240406383167612, "Classification": 0.016894315589557996, "Regression_pt": 0.1449972152709961, "Regression_eta": 0.018218393759293988, "Regression_sin_phi": 0.015461309389634565, "Regression_cos_phi": 0.015599595416675915, "Regression_energy": 0.15245149785822087, "Total": 5.359342817826705, "group/cld/Classification": 0.015288775271557748, "group/cld/Classification_binary": 2.23119159454995, "group/cld/Regression_cos_phi": 0.007274310005472061, "group/cld/Regression_energy": 0.1318628838721742, "group/cld/Regression_eta": 0.012618192966948164, "group/cld/Regression_pt": 0.1275473249719498, "group/cld/Regression_sin_phi": 0.00670947419836166, "group/cld/Total": 4.79919888110871, "group/cld/Total_unweighted": 2.5324925558364133, "group/clic/Classification": 0.01737492008412138, "group/clic/Classification_binary": 2.2084980416805187, "group/clic/Regression_cos_phi": 0.00799315280102669, "group/clic/Regression_energy": 0.1240154225775536, "group/clic/Regression_eta": 0.01087617240053542, "group/clic/Regression_pt": 0.1211888942312687, "group/clic/Regression_sin_phi": 0.007901298872967984, "group/clic/Total": 4.703741844664228, "group/clic/Total_unweighted": 2.497847902647993, "group/colliderml/Classification": 0.020198805257678032, "group/colliderml/Classification_binary": 2.2486891746520996, "group/colliderml/Regression_cos_phi": 0.06239904463291168, "group/colliderml/Regression_energy": 0.2964613437652588, "group/colliderml/Regression_eta": 0.05623674392700195, "group/colliderml/Regression_pt": 0.2661929726600647, "group/colliderml/Regression_sin_phi": 0.06337732821702957, "group/colliderml/Total": 8.930601119995117, "group/colliderml/Total_unweighted": 3.0135554131120443, "Total_unweighted": 2.5876629656011407, "metrics/particle/class_1/efficiency": 0.9646328091621399, "metrics/particle/class_1/pid_accuracy": 0.9836462140083313, "metrics/particle/class_2/efficiency": 0.2983149588108063, "metrics/particle/class_2/pid_accuracy": 0.7251419425010681, "metrics/particle/class_3/efficiency": 0.664543092250824, "metrics/particle/class_3/pid_accuracy": 0.9771265387535095, "metrics/particle/class_4/efficiency": 0.9225857853889465, "metrics/particle/class_4/pid_accuracy": 0.5337370038032532, "metrics/particle/class_5/efficiency": 0.9826839566230774, "metrics/particle/class_5/pid_accuracy": 0.7048457860946655, "metrics/particle/count/bias_mean": -0.9811428785324097, "metrics/particle/count/mae": 4.518285751342773, "metrics/particle/count/prediction_mean": 91.17314147949219, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04406086355447769, "metrics/particle/event/energy_response_mean": 0.9970048666000366, "metrics/particle/event/met_abs_error": 5.161143779754639, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03488130122423172, "metrics/particle/event/scalar_pt_response_mean": 1.0055674314498901, "metrics/particle/event/vector_pt_closure": 0.037783633917570114, "metrics/particle/matched/delta_eta_abs_mean": 0.01244688406586647, "metrics/particle/matched/delta_phi_abs_mean": 0.011997491121292114, "metrics/particle/matched/delta_r_mean": 0.01938646472990513, "metrics/particle/matched/energy_relative_abs_mean": 0.08774060010910034, "metrics/particle/matched/pid_accuracy": 0.9600940346717834, "metrics/particle/matched/pt_relative_abs_mean": 0.08308474719524384, "metrics/particle/matching/duplicate_fraction": 0.033098720014095306, "metrics/particle/matching/efficiency": 0.7360575199127197, "metrics/particle/matching/f1": 0.7399967908859253, "metrics/particle/matching/purity": 0.7439785003662109, "metrics/particle/matching/target_coverage_dr0p05": 0.6822471618652344, "metrics/particle/matching/target_coverage_dr0p10": 0.7698827981948853, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.660091757774353, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8137719631195068}, "step_train_time": 14.332003355026245, "step_valid_time": 13.25813798501622, "step_total_time": 48273.76965904236, "elapsed_training_loop_seconds": 48273.76965904236, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0010480149451775, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0658589183879863, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1434, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.034993270524899055, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9650067294751009, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048638578103197, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0758127429910761, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9553172475424486, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9557443004023245, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044682752457551385, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9543828264758497, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9535299374441466, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.953956191327671, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561717352415027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0086750528851323, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07864174992073858, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 617, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 574, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 573, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9140127388535032, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9303079416531604, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9220883534136546, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06969205834683954, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9124203821656051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9286871961102107, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9204819277108434, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9140127388535032, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004852718599837, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.051773577254080605, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1427, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1423, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9564343163538874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9564343163538874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9564343163538874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0435656836461126, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9537533512064343, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9537533512064343, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9537533512064343, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9564343163538874, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031706347461253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06867083092702186, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1101, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1044, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1041, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9396939693969397, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9482288828337875, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9439421338155516, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.051771117166212535, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.936993699369937, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9455040871934605, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9412296564195298, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9396939693969397, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0070368021735023, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06925438679549911, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 638, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 585, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 581, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9198113207547169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9169278996865203, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9183673469387755, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08307210031347963, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9135220125786163, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9106583072100314, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9120879120879121, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9198113207547169, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9940104673890161, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12901278120514803, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3546, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2679, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2653, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7552861573160418, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7554991539763113, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7553926406316086, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24450084602368866, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7479560191711305, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7481669486745629, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.748061469053997, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7552861573160418, "step_test_time": 37.22253853396978, "step_plot_time": 26.957648783922195, "step_validation_cycle_time": 78.14504016295541} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_380000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_380000.json new file mode 100644 index 0000000000000000000000000000000000000000..06590670350198b695228099b2ba20d1462e2e9c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_380000.json @@ -0,0 +1 @@ +{"epoch": 0.9644302881869979, "train": {"Classification_binary": 2.2224833965301514, "Classification": 0.012258302420377731, "Regression_pt": 0.11782650649547577, "Regression_eta": 0.007178879342973232, "Regression_sin_phi": 0.006560474634170532, "Regression_cos_phi": 0.00783612672239542, "Regression_energy": 0.11774472147226334, "Total": 4.534394264221191, "Total_unweighted": 2.4918884076178074}, "train_group": {"cld": {"Classification_binary": 2.236689242872748, "Classification": 0.01564165731673842, "Regression_pt": 0.1229752269951073, "Regression_eta": 0.011733743256900471, "Regression_sin_phi": 0.007058673432340157, "Regression_cos_phi": 0.007182391450248167, "Regression_energy": 0.12732618172336985, "Total": 4.732768295763491, "Total_unweighted": 2.5286071170474522}, "clic": {"Classification_binary": 2.138801415166455, "Classification": 0.016784161117351697, "Regression_pt": 0.11529704066227653, "Regression_eta": 0.009265731718296375, "Regression_sin_phi": 0.008326334917839772, "Regression_cos_phi": 0.008464922821762677, "Regression_energy": 0.11539137829652803, "Total": 4.513039720272589, "Total_unweighted": 2.4123309847005103}, "colliderml": {"Classification_binary": 2.2502705191706753, "Classification": 0.020423448580252874, "Regression_pt": 0.2650701074718355, "Regression_eta": 0.057667689190970525, "Regression_sin_phi": 0.06405018866285905, "Regression_cos_phi": 0.0639036594926416, "Regression_energy": 0.30107746347113773, "Total": 8.988779609267777, "Total_unweighted": 3.0224630760403723}}, "valid": {"Classification_binary": 2.199542236328125, "Classification": 0.01683856248855591, "Regression_pt": 0.14357131611217153, "Regression_eta": 0.017816146937283603, "Regression_sin_phi": 0.015306489034132524, "Regression_cos_phi": 0.015486982735720549, "Regression_energy": 0.151044377413663, "Total": 5.305247913707387, "group/cld/Classification": 0.015319604822929869, "group/cld/Classification_binary": 2.199715147627161, "group/cld/Regression_cos_phi": 0.007134640470464179, "group/cld/Regression_energy": 0.13078022003173828, "group/cld/Regression_eta": 0.012198534417659678, "group/cld/Regression_pt": 0.1261597085506358, "group/cld/Regression_sin_phi": 0.006629471449141807, "group/cld/Total": 4.742280676009807, "group/cld/Total_unweighted": 2.4979373273697303, "group/clic/Classification": 0.017237084977170254, "group/clic/Classification_binary": 2.185122875457114, "group/clic/Regression_cos_phi": 0.007949856367517025, "group/clic/Regression_energy": 0.12205386669077772, "group/clic/Regression_eta": 0.010413784930046568, "group/clic/Regression_pt": 0.11930854269798766, "group/clic/Regression_sin_phi": 0.0077778202422121735, "group/clic/Total": 4.642412875561004, "group/clic/Total_unweighted": 2.4698638313628254, "group/colliderml/Classification": 0.020129818469285965, "group/colliderml/Classification_binary": 2.2413928508758545, "group/colliderml/Regression_cos_phi": 0.06216229125857353, "group/colliderml/Regression_energy": 0.29572975635528564, "group/colliderml/Regression_eta": 0.05606235936284065, "group/colliderml/Regression_pt": 0.26598984003067017, "group/colliderml/Regression_sin_phi": 0.06291069090366364, "group/colliderml/Total": 8.906037330627441, "group/colliderml/Total_unweighted": 3.004377607256174, "Total_unweighted": 2.5596061110496526, "metrics/particle/class_1/efficiency": 0.9629795551300049, "metrics/particle/class_1/pid_accuracy": 0.9827196002006531, "metrics/particle/class_2/efficiency": 0.3013550043106079, "metrics/particle/class_2/pid_accuracy": 0.7351203560829163, "metrics/particle/class_3/efficiency": 0.6730374097824097, "metrics/particle/class_3/pid_accuracy": 0.9792419075965881, "metrics/particle/class_4/efficiency": 0.9098164439201355, "metrics/particle/class_4/pid_accuracy": 0.5535087585449219, "metrics/particle/class_5/efficiency": 0.9769119620323181, "metrics/particle/class_5/pid_accuracy": 0.740029513835907, "metrics/particle/count/bias_mean": -0.694857120513916, "metrics/particle/count/mae": 4.097142696380615, "metrics/particle/count/prediction_mean": 91.45942687988281, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04605543613433838, "metrics/particle/event/energy_response_mean": 1.013875126838684, "metrics/particle/event/met_abs_error": 5.168857097625732, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0350896492600441, "metrics/particle/event/scalar_pt_response_mean": 1.0071276426315308, "metrics/particle/event/vector_pt_closure": 0.03777178376913071, "metrics/particle/matched/delta_eta_abs_mean": 0.012056172825396061, "metrics/particle/matched/delta_phi_abs_mean": 0.012129753828048706, "metrics/particle/matched/delta_r_mean": 0.019149212166666985, "metrics/particle/matched/energy_relative_abs_mean": 0.09041190147399902, "metrics/particle/matched/pid_accuracy": 0.9614272117614746, "metrics/particle/matched/pt_relative_abs_mean": 0.08276354521512985, "metrics/particle/matching/duplicate_fraction": 0.03252652287483215, "metrics/particle/matching/efficiency": 0.7394741773605347, "metrics/particle/matching/f1": 0.7422726154327393, "metrics/particle/matching/purity": 0.7450922727584839, "metrics/particle/matching/target_coverage_dr0p05": 0.6933279633522034, "metrics/particle/matching/target_coverage_dr0p10": 0.7738450765609741, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6640850901603699, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8156569600105286}, "step_train_time": 13.049356460571289, "step_valid_time": 13.495833751978353, "step_total_time": 49611.66856741905, "elapsed_training_loop_seconds": 49611.66856741905, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0049365986183916, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06198941183296203, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9569023569023569, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9565802760013463, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0430976430976431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9542395693135935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9548821548821549, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9545607539548974, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9562584118438762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0059752152727035, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07366301929854913, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1127, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1074, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.960644007155635, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9529724933451642, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.956792873051225, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04702750665483585, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9485359361135759, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9523385300668151, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.960644007155635, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0107572599200074, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07339743614705674, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07802547770700637, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9219745222929936, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028362865983906, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05218746701763388, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1501, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1437, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9631367292225201, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9573617588274483, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9602405613097227, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04263824117255163, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9597855227882037, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540306462358428, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9568994320080187, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9631367292225201, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036742874888347, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07050194688035893, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1104, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1053, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1051, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9477947794779478, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9538043478260869, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9507900677200903, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04619565217391304, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.945994599459946, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9519927536231884, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9489841986455982, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9477947794779478, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0051871385598803, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06790004646853565, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 634, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 585, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 583, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9198113207547169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9227129337539433, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9212598425196851, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07728706624605679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9166666666666666, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.919558359621451, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9181102362204724, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9198113207547169, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000431866910351, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1270705884986405, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3630, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2703, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2672, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7620524386805752, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7446280991735538, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7532395151177372, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2553719008264463, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7533126585847195, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7360881542699724, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7446008081371046, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7620524386805752, "step_test_time": 37.32674858800601, "step_plot_time": 27.01525886100717, "step_validation_cycle_time": 78.56982851901557} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_390000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_390000.json new file mode 100644 index 0000000000000000000000000000000000000000..5b93dad3afedcfa92b3da94f1a285b0b26f5e1e9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_390000.json @@ -0,0 +1 @@ +{"epoch": 0.9898100326129716, "train": {"Classification_binary": 2.180967092514038, "Classification": 0.019368883222341537, "Regression_pt": 0.2725057303905487, "Regression_eta": 0.05183546617627144, "Regression_sin_phi": 0.059079043567180634, "Regression_cos_phi": 0.06034050136804581, "Regression_energy": 0.2923857569694519, "Total": 8.784332275390625, "Total_unweighted": 2.936482474207878}, "train_group": {"cld": {"Classification_binary": 2.205826662682198, "Classification": 0.01550277535364524, "Regression_pt": 0.12255314050673007, "Regression_eta": 0.01319885098685821, "Regression_sin_phi": 0.007041010632621567, "Regression_cos_phi": 0.006899407431315463, "Regression_energy": 0.12741688923076824, "Total": 4.701486984172741, "Total_unweighted": 2.498438736824137}, "clic": {"Classification_binary": 2.1420509324775443, "Classification": 0.016707156328959867, "Regression_pt": 0.11488780573950158, "Regression_eta": 0.010219854253061765, "Regression_sin_phi": 0.008364857669483434, "Regression_cos_phi": 0.008182213393490281, "Regression_energy": 0.11569352349540493, "Total": 4.515901143844421, "Total_unweighted": 2.416106343357446}, "colliderml": {"Classification_binary": 2.248612960655532, "Classification": 0.020440621903824237, "Regression_pt": 0.2649587075599653, "Regression_eta": 0.0577545794630479, "Regression_sin_phi": 0.06406086426741349, "Regression_cos_phi": 0.06434828112136104, "Regression_energy": 0.30091308030539643, "Total": 8.991108651646597, "Total_unweighted": 3.021089095276541}}, "valid": {"Classification_binary": 2.1956592906605112, "Classification": 0.016810651258988812, "Regression_pt": 0.1432141217318448, "Regression_eta": 0.017588819156993517, "Regression_sin_phi": 0.015291106700897217, "Regression_cos_phi": 0.015478950197046454, "Regression_energy": 0.1506250728260387, "Total": 5.293108021129261, "group/cld/Classification": 0.015314270841314438, "group/cld/Classification_binary": 2.1931856845287565, "group/cld/Regression_cos_phi": 0.007185426164180674, "group/cld/Regression_energy": 0.12999745632739776, "group/cld/Regression_eta": 0.011941410125570094, "group/cld/Regression_pt": 0.1255644128677693, "group/cld/Regression_sin_phi": 0.006646550716237819, "group/cld/Total": 4.724003893263797, "group/cld/Total_unweighted": 2.489835211571226, "group/clic/Classification": 0.017203376648273875, "group/clic/Classification_binary": 2.1861770305227726, "group/clic/Regression_cos_phi": 0.007905946132984567, "group/clic/Regression_energy": 0.1229001207554594, "group/clic/Regression_eta": 0.01019299918032707, "group/clic/Regression_pt": 0.12016777282065534, "group/clic/Regression_sin_phi": 0.007810522901250961, "group/clic/Total": 4.655587378968584, "group/clic/Total_unweighted": 2.4723577689617233, "group/colliderml/Classification": 0.020052608102560043, "group/colliderml/Classification_binary": 2.230776786804199, "group/colliderml/Regression_cos_phi": 0.06208688020706177, "group/colliderml/Regression_energy": 0.29266074299812317, "group/colliderml/Regression_eta": 0.05590331554412842, "group/colliderml/Regression_pt": 0.26275888085365295, "group/colliderml/Regression_sin_phi": 0.06265871226787567, "group/colliderml/Total": 8.837566375732422, "group/colliderml/Total_unweighted": 2.9868979267776012, "Total_unweighted": 2.5546680125323213, "metrics/particle/class_1/efficiency": 0.9633349180221558, "metrics/particle/class_1/pid_accuracy": 0.984185516834259, "metrics/particle/class_2/efficiency": 0.30231043696403503, "metrics/particle/class_2/pid_accuracy": 0.7332279682159424, "metrics/particle/class_3/efficiency": 0.6724786162376404, "metrics/particle/class_3/pid_accuracy": 0.979016900062561, "metrics/particle/class_4/efficiency": 0.90343177318573, "metrics/particle/class_4/pid_accuracy": 0.4805653691291809, "metrics/particle/class_5/efficiency": 0.9797979593276978, "metrics/particle/class_5/pid_accuracy": 0.7290132641792297, "metrics/particle/count/bias_mean": -1.063428521156311, "metrics/particle/count/mae": 4.3331427574157715, "metrics/particle/count/prediction_mean": 91.09085845947266, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04327686131000519, "metrics/particle/event/energy_response_mean": 0.9931787252426147, "metrics/particle/event/met_abs_error": 4.909048080444336, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03396428003907204, "metrics/particle/event/scalar_pt_response_mean": 0.9984498620033264, "metrics/particle/event/vector_pt_closure": 0.03590749204158783, "metrics/particle/matched/delta_eta_abs_mean": 0.011125830933451653, "metrics/particle/matched/delta_phi_abs_mean": 0.011918163858354092, "metrics/particle/matched/delta_r_mean": 0.018258677795529366, "metrics/particle/matched/energy_relative_abs_mean": 0.0882626548409462, "metrics/particle/matched/pid_accuracy": 0.9612172245979309, "metrics/particle/matched/pt_relative_abs_mean": 0.08311790972948074, "metrics/particle/matching/duplicate_fraction": 0.031528960913419724, "metrics/particle/matching/efficiency": 0.739467978477478, "metrics/particle/matching/f1": 0.7437593340873718, "metrics/particle/matching/purity": 0.7481008172035217, "metrics/particle/matching/target_coverage_dr0p05": 0.6961803436279297, "metrics/particle/matching/target_coverage_dr0p10": 0.7729088068008423, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6632913947105408, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8169529438018799}, "step_train_time": 9.965872764587402, "step_valid_time": 13.591960534104146, "step_total_time": 50933.44411492348, "elapsed_training_loop_seconds": 50933.44411492348, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9990397441160228, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06948902373155696, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9569023569023569, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9565802760013463, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0430976430976431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9515477792732167, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9521885521885521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9518680578929654, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9562584118438762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994094534070791, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07313003907075111, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1124, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1076, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1071, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.962432915921288, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9572953736654805, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9598572702943801, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042704626334519574, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9579606440071556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.952846975088968, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9553969669937555, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.962432915921288, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0033965708066166, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07076186069668211, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 623, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9331210191082803, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9406099518459069, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9368505195843325, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0593900481540931, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9331210191082803, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9406099518459069, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9368505195843325, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9331210191082803, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.997933288895503, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05373488473960375, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1490, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1435, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1433, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9617962466487936, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9630872483221476, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9624413145539906, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03691275167785235, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960455764075067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.961744966442953, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9610999329309189, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9617962466487936, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9986693030450382, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06523092958979615, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1100, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1042, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1040, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9378937893789379, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9472727272727273, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9425599276345545, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05272727272727273, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9360936093609361, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9454545454545454, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9407507914970602, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9378937893789379, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008297916217537, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06400287736624222, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 641, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 593, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 592, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9251170046801872, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9287392325763508, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0748829953198128, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9308176100628931, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9235569422776911, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9271730618637432, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9323899371069182, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9897933782788575, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12176911104540378, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3595, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2698, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2667, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7606427967296306, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7504867872044506, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7555306636796416, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24951321279554936, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.751903016633775, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7418636995827538, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7468496219546346, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7606427967296306, "step_test_time": 37.362146836007014, "step_plot_time": 27.137136231991462, "step_validation_cycle_time": 78.83067700895481} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..fe5937b8faf4d762e4cd2728e480dcd0637b8cc3 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 0.10151897770389452, "train": {"Classification_binary": 3.596055269241333, "Classification": 0.0339520089328289, "Regression_pt": 0.23345167934894562, "Regression_eta": 0.021893171593546867, "Regression_sin_phi": 0.011437210254371166, "Regression_cos_phi": 0.012016302905976772, "Regression_energy": 0.22861248254776, "Total": 8.261199951171875, "Total_unweighted": 4.137418124824762}, "train_group": {"cld": {"Classification_binary": 3.7975161304017027, "Classification": 0.029635506431142728, "Regression_pt": 0.2421122963824672, "Regression_eta": 0.02439154336831259, "Regression_sin_phi": 0.012469693501814531, "Regression_cos_phi": 0.012634791850675633, "Regression_energy": 0.2511829322951283, "Total": 8.671559323807676, "Total_unweighted": 4.369942894231244}, "clic": {"Classification_binary": 3.4781831699806647, "Classification": 0.03169118962040893, "Regression_pt": 0.19849926215392333, "Regression_eta": 0.017685332931242546, "Regression_sin_phi": 0.013883735341307995, "Regression_cos_phi": 0.01390964645674234, "Regression_energy": 0.199341758660547, "Total": 7.604772805451631, "Total_unweighted": 3.953194095144837}, "colliderml": {"Classification_binary": 2.974757954880998, "Classification": 0.030146584502077318, "Regression_pt": 0.38747287530440827, "Regression_eta": 0.06574310374644783, "Regression_sin_phi": 0.07520570319604587, "Regression_cos_phi": 0.07480505563027866, "Regression_energy": 0.4347141876414016, "Total": 12.237674770412502, "Total_unweighted": 4.042845464901657}}, "valid": {"Classification_binary": 3.538695456764915, "Classification": 0.03111335580999201, "Regression_pt": 0.25021114349365237, "Regression_eta": 0.026875066757202148, "Regression_sin_phi": 0.021081538633866744, "Regression_cos_phi": 0.02175385301763361, "Regression_energy": 0.26112715981223367, "Total": 8.806033047762783, "group/cld/Classification": 0.02972522187740245, "group/cld/Classification_binary": 3.784961781603225, "group/cld/Regression_cos_phi": 0.012691972103524716, "group/cld/Regression_energy": 0.25768162341828044, "group/cld/Regression_eta": 0.023013779457579268, "group/cld/Regression_pt": 0.24859026645092255, "group/cld/Regression_sin_phi": 0.011085366949122002, "group/cld/Total": 8.742114127950465, "group/cld/Total_unweighted": 4.367750011860056, "group/clic/Classification": 0.032993042722661445, "group/clic/Classification_binary": 3.495615208402593, "group/clic/Regression_cos_phi": 0.013469084780266944, "group/clic/Regression_energy": 0.20893447957140335, "group/clic/Regression_eta": 0.01835953174753392, "group/clic/Regression_pt": 0.20515681327657498, "group/clic/Regression_sin_phi": 0.01333510495246725, "group/clic/Total": 7.7600532694065825, "group/clic/Total_unweighted": 3.9878632654535004, "group/colliderml/Classification": 0.02966945618391037, "group/colliderml/Classification_binary": 2.941831111907959, "group/colliderml/Regression_cos_phi": 0.07270966470241547, "group/colliderml/Regression_energy": 0.42456430196762085, "group/colliderml/Regression_eta": 0.06323200464248657, "group/colliderml/Regression_pt": 0.38731977343559265, "group/colliderml/Regression_sin_phi": 0.07320043444633484, "group/colliderml/Total": 12.066361427307129, "group/colliderml/Total_unweighted": 3.9925267472863197, "Total_unweighted": 4.150857574289495, "metrics/particle/class_1/efficiency": 0.917600154876709, "metrics/particle/class_1/pid_accuracy": 0.9708358645439148, "metrics/particle/class_2/efficiency": 0.1785373091697693, "metrics/particle/class_2/pid_accuracy": 0.3043055236339569, "metrics/particle/class_3/efficiency": 0.521787703037262, "metrics/particle/class_3/pid_accuracy": 0.9596765637397766, "metrics/particle/class_4/efficiency": 0.8483639359474182, "metrics/particle/class_4/pid_accuracy": 0.041392285376787186, "metrics/particle/class_5/efficiency": 0.9567099809646606, "metrics/particle/class_5/pid_accuracy": 0.05429864302277565, "metrics/particle/count/bias_mean": -0.349142849445343, "metrics/particle/count/mae": 5.706857204437256, "metrics/particle/count/prediction_mean": 91.80514526367188, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.07962708920240402, "metrics/particle/event/energy_response_mean": 1.0162272453308105, "metrics/particle/event/met_abs_error": 11.070085525512695, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07021187990903854, "metrics/particle/event/scalar_pt_response_mean": 1.0282901525497437, "metrics/particle/event/vector_pt_closure": 0.07668830454349518, "metrics/particle/matched/delta_eta_abs_mean": 0.024781158193945885, "metrics/particle/matched/delta_phi_abs_mean": 0.019430533051490784, "metrics/particle/matched/delta_r_mean": 0.03504467383027077, "metrics/particle/matched/energy_relative_abs_mean": 0.10095146298408508, "metrics/particle/matched/pid_accuracy": 0.9245025515556335, "metrics/particle/matched/pt_relative_abs_mean": 0.09767644107341766, "metrics/particle/matching/duplicate_fraction": 0.049066655337810516, "metrics/particle/matching/efficiency": 0.63603276014328, "metrics/particle/matching/f1": 0.6372398734092712, "metrics/particle/matching/purity": 0.6384516358375549, "metrics/particle/matching/target_coverage_dr0p05": 0.49885284900665283, "metrics/particle/matching/target_coverage_dr0p10": 0.6755379438400269, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5759285688400269, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7441681623458862}, "step_train_time": 7.3326215744018555, "step_valid_time": 12.359727187897079, "step_total_time": 4967.083684921265, "elapsed_training_loop_seconds": 4967.083684921265, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0095605492990156, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1647398265960085, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1499, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1356, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1321, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9125168236877523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.904603068712475, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9085427135678392, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09539693128752502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8889636608344549, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8812541694462975, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8850921273031825, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9125168236877523, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0089103861720807, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18199941010663756, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1138, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 974, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 954, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8711985688729875, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8558875219683656, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8634751773049646, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.14411247803163443, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.853309481216458, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8383128295254832, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8457446808510638, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8711985688729875, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.014992224446434, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17121471870901606, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 494, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 482, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7866242038216561, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7853736089030207, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7859984089101034, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.21462639109697934, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.767515923566879, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.766295707472178, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7669053301511536, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7866242038216561, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0112302284007695, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12815517907090213, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1502, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1365, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1346, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9148793565683646, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9087882822902796, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9118236472945892, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09121171770972038, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9021447721179625, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8961384820239681, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8991315965263861, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9148793565683646, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0117480903649776, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16292609771736033, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1092, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 963, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 952, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8667866786678667, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8818681318681318, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8742623694961417, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11813186813186813, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8568856885688569, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8717948717948718, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8642759872900591, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8667866786678667, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0112904358531005, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15441102238551996, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 650, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 528, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 517, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8301886792452831, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8123076923076923, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8211508553654744, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.18769230769230769, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8128930817610063, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7953846153846154, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8040435458786936, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8301886792452831, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0668894966574343, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2962409905718314, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3731, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2527, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2331, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7124330420073302, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.6772983114446529, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.6944215443803242, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.3227016885553471, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6571750775303073, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.624765478424015, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.640560593569662, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7124330420073302, "step_test_time": 33.29004814696964, "step_plot_time": 26.49475426506251, "step_validation_cycle_time": 72.65341639495455} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_400000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_400000.json new file mode 100644 index 0000000000000000000000000000000000000000..d18e413573edd2dea4a4969dd9b482bf65ab659e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_400000.json @@ -0,0 +1 @@ +{"epoch": 1.0151897770389453, "train": {"Classification_binary": 2.0160083770751953, "Classification": 0.01671995036303997, "Regression_pt": 0.10708055645227432, "Regression_eta": 0.009245837107300758, "Regression_sin_phi": 0.006046317517757416, "Regression_cos_phi": 0.00562786590307951, "Regression_energy": 0.10420416295528412, "Total": 4.173581600189209, "Total_unweighted": 2.2649330673739314}, "train_group": {"cld": {"Classification_binary": 2.1995555130307545, "Classification": 0.015685298767297746, "Regression_pt": 0.12265086840994344, "Regression_eta": 0.013431886885828245, "Regression_sin_phi": 0.006977885070641687, "Regression_cos_phi": 0.00720823436632507, "Regression_energy": 0.1277192942351995, "Total": 4.704887258792352, "Total_unweighted": 2.4932289807659904}, "clic": {"Classification_binary": 2.1292178283582577, "Classification": 0.01654782411348712, "Regression_pt": 0.11284713185316808, "Regression_eta": 0.010050060695272174, "Regression_sin_phi": 0.008452056848385313, "Regression_cos_phi": 0.008272965978439505, "Regression_energy": 0.11356261111907773, "Total": 4.468032742405797, "Total_unweighted": 2.398950478966088}, "colliderml": {"Classification_binary": 2.2367537465539424, "Classification": 0.020262084341979958, "Regression_pt": 0.26902370159332456, "Regression_eta": 0.057349986209346726, "Regression_sin_phi": 0.0636473827780009, "Regression_cos_phi": 0.06393139926834149, "Regression_energy": 0.3047861159295291, "Total": 9.029793073465159, "Total_unweighted": 3.015754416674465}}, "valid": {"Classification_binary": 2.18875302401456, "Classification": 0.016755177757956766, "Regression_pt": 0.14246111783114346, "Regression_eta": 0.017668009888042104, "Regression_sin_phi": 0.015281807292591442, "Regression_cos_phi": 0.015418249910528009, "Regression_energy": 0.15000133514404296, "Total": 5.273585094105114, "group/cld/Classification": 0.01512946220154458, "group/cld/Classification_binary": 2.1824486915101398, "group/cld/Regression_cos_phi": 0.00718833030538356, "group/cld/Regression_energy": 0.1294393133609853, "group/cld/Regression_eta": 0.011915943724043827, "group/cld/Regression_pt": 0.12466568642474235, "group/cld/Regression_sin_phi": 0.006725279574698591, "group/cld/Total": 4.699108042615525, "group/cld/Total_unweighted": 2.4775127071015377, "group/clic/Classification": 0.017289771678599904, "group/clic/Classification_binary": 2.181826490036985, "group/clic/Regression_cos_phi": 0.00783538374494999, "group/clic/Regression_energy": 0.12252299329067798, "group/clic/Regression_eta": 0.010401999696772149, "group/clic/Regression_pt": 0.11985549520938954, "group/clic/Regression_sin_phi": 0.007710955244429568, "group/clic/Total": 4.646035701670545, "group/clic/Total_unweighted": 2.467443088901805, "group/colliderml/Classification": 0.019960306584835052, "group/colliderml/Classification_binary": 2.2276203632354736, "group/colliderml/Regression_cos_phi": 0.06186831742525101, "group/colliderml/Regression_energy": 0.29111984372138977, "group/colliderml/Regression_eta": 0.05590863153338432, "group/colliderml/Regression_pt": 0.2611392140388489, "group/colliderml/Regression_sin_phi": 0.0626559853553772, "group/colliderml/Total": 8.804533958435059, "group/colliderml/Total_unweighted": 2.98027266189456, "Total_unweighted": 2.5463387218388642, "metrics/particle/class_1/efficiency": 0.9636130332946777, "metrics/particle/class_1/pid_accuracy": 0.9842221736907959, "metrics/particle/class_2/efficiency": 0.30105099081993103, "metrics/particle/class_2/pid_accuracy": 0.7311021089553833, "metrics/particle/class_3/efficiency": 0.671430766582489, "metrics/particle/class_3/pid_accuracy": 0.9788177013397217, "metrics/particle/class_4/efficiency": 0.9082202911376953, "metrics/particle/class_4/pid_accuracy": 0.5184534192085266, "metrics/particle/class_5/efficiency": 0.9711399674415588, "metrics/particle/class_5/pid_accuracy": 0.7013372778892517, "metrics/particle/count/bias_mean": -1.7011429071426392, "metrics/particle/count/mae": 4.506856918334961, "metrics/particle/count/prediction_mean": 90.45314025878906, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04295717924833298, "metrics/particle/event/energy_response_mean": 0.9895240068435669, "metrics/particle/event/met_abs_error": 4.906898021697998, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03374280780553818, "metrics/particle/event/scalar_pt_response_mean": 0.9959083199501038, "metrics/particle/event/vector_pt_closure": 0.03561228886246681, "metrics/particle/matched/delta_eta_abs_mean": 0.011252249591052532, "metrics/particle/matched/delta_phi_abs_mean": 0.011274084448814392, "metrics/particle/matched/delta_r_mean": 0.017856717109680176, "metrics/particle/matched/energy_relative_abs_mean": 0.08726339787244797, "metrics/particle/matched/pid_accuracy": 0.9612731337547302, "metrics/particle/matched/pt_relative_abs_mean": 0.0827101320028305, "metrics/particle/matching/duplicate_fraction": 0.03038668818771839, "metrics/particle/matching/efficiency": 0.7389346957206726, "metrics/particle/matching/f1": 0.7458184957504272, "metrics/particle/matching/purity": 0.752831757068634, "metrics/particle/matching/target_coverage_dr0p05": 0.6965213418006897, "metrics/particle/matching/target_coverage_dr0p10": 0.7718732357025146, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6623178720474243, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8141191601753235}, "step_train_time": 17.23278832435608, "step_valid_time": 13.468053968041204, "step_total_time": 52277.529732227325, "elapsed_training_loop_seconds": 52277.529732227325, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008061998959075, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06141306535013391, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1489, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1426, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9629878869448183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9610476830087307, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9620168067226891, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03895231699126931, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9596231493943472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9576897246474144, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.958655462184874, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9629878869448183, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015384159227652, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0704637165734795, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1127, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1075, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9651162790697675, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9574090505767524, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9612472160356348, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04259094942324756, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9615384615384616, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9538598047914818, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9576837416481069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9651162790697675, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0027667620344567, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07539158430613824, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 584, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 583, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9299363057324841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9358974358974359, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9329073482428115, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0641025641025641, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.928343949044586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9342948717948718, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9313099041533547, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9299363057324841, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9997301462509455, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05214568486784332, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1496, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1437, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1433, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9631367292225201, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9605614973262032, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9618473895582329, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03943850267379679, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960455764075067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9578877005347594, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9591700133868809, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9631367292225201, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9989090031704532, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06388794511275964, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1051, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1050, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.945994599459946, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9528558476881233, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9494128274616079, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0471441523118767, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9450945094509451, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9519492293744334, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.948509485094851, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.945994599459946, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030919245372045, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06494094503338532, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 638, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 590, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 585, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9247648902821317, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9262166405023547, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07523510971786834, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9198113207547169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9169278996865203, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9183673469387755, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9276729559748428, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9871245946733138, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12063233865796896, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3571, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2713, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2683, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.764871722582464, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7597311677401288, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7622927788704692, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2402688322598712, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7564138708767973, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7513301596191543, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7538634447878617, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.764871722582464, "step_test_time": 37.846125572919846, "step_plot_time": 27.194124158937484, "step_validation_cycle_time": 79.25816290907096} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_410000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_410000.json new file mode 100644 index 0000000000000000000000000000000000000000..3363189386fb4925bd54976bf6b431534dd73abe --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_410000.json @@ -0,0 +1 @@ +{"epoch": 1.0405695214649189, "train": {"Classification_binary": 1.797808289527893, "Classification": 0.018798548728227615, "Regression_pt": 0.0668778270483017, "Regression_eta": 0.003247114596888423, "Regression_sin_phi": 0.00870705209672451, "Regression_cos_phi": 0.0045224628411233425, "Regression_energy": 0.06824995577335358, "Total": 3.336625099182129, "Total_unweighted": 1.9682112506125122}, "train_group": {"cld": {"Classification_binary": 2.2054072106564724, "Classification": 0.015362455305439216, "Regression_pt": 0.12240931528198111, "Regression_eta": 0.01437639335433284, "Regression_sin_phi": 0.00707694443709052, "Regression_cos_phi": 0.007350781649300778, "Regression_energy": 0.1284759111098341, "Total": 4.717381105050668, "Total_unweighted": 2.500459011794451}, "clic": {"Classification_binary": 2.10686618981961, "Classification": 0.016458701989250982, "Regression_pt": 0.11235702307727523, "Regression_eta": 0.009280942376942826, "Regression_sin_phi": 0.008262401098499777, "Regression_cos_phi": 0.008067908587385928, "Regression_energy": 0.11283573160271444, "Total": 4.425616499906528, "Total_unweighted": 2.374128898551679}, "colliderml": {"Classification_binary": 2.2334874198959396, "Classification": 0.020147188049223688, "Regression_pt": 0.26550164621871514, "Regression_eta": 0.057449453720101365, "Regression_sin_phi": 0.06399842500462904, "Regression_cos_phi": 0.06403667729224886, "Regression_energy": 0.3020002593596776, "Total": 8.978195890650019, "Total_unweighted": 3.0066210695405355}}, "valid": {"Classification_binary": 2.179015419699929, "Classification": 0.016670574925162576, "Regression_pt": 0.14113839756358754, "Regression_eta": 0.017533650181510232, "Regression_sin_phi": 0.015250458500602028, "Regression_cos_phi": 0.015386395020918413, "Regression_energy": 0.14851668964732778, "Total": 5.238418301669034, "group/cld/Classification": 0.015199834996081412, "group/cld/Classification_binary": 2.169856457000083, "group/cld/Regression_cos_phi": 0.007118041845078164, "group/cld/Regression_energy": 0.1280343989108471, "group/cld/Regression_eta": 0.01179446945799158, "group/cld/Regression_pt": 0.1236580990730448, "group/cld/Regression_sin_phi": 0.006642432923012591, "group/cld/Total": 4.665801677298038, "group/cld/Total_unweighted": 2.462303734206139, "group/clic/Classification": 0.01701848050381275, "group/clic/Classification_binary": 2.172643783244681, "group/clic/Regression_cos_phi": 0.007818303209670047, "group/clic/Regression_energy": 0.12068149891305477, "group/clic/Regression_eta": 0.010236911317135425, "group/clic/Regression_pt": 0.11791808554466734, "group/clic/Regression_sin_phi": 0.007733182704195063, "group/clic/Total": 4.601157330452128, "group/clic/Total_unweighted": 2.4540502454372164, "group/colliderml/Classification": 0.019968939945101738, "group/colliderml/Classification_binary": 2.224637985229492, "group/colliderml/Regression_cos_phi": 0.06190590560436249, "group/colliderml/Regression_energy": 0.2904494106769562, "group/colliderml/Regression_eta": 0.05582664906978607, "group/colliderml/Regression_pt": 0.26069650053977966, "group/colliderml/Regression_sin_phi": 0.06261852383613586, "group/colliderml/Total": 8.792434692382812, "group/colliderml/Total_unweighted": 2.976103914901614, "Total_unweighted": 2.5335115855390375, "metrics/particle/class_1/efficiency": 0.9637366533279419, "metrics/particle/class_1/pid_accuracy": 0.9845448136329651, "metrics/particle/class_2/efficiency": 0.2988795340061188, "metrics/particle/class_2/pid_accuracy": 0.721447229385376, "metrics/particle/class_3/efficiency": 0.6714168190956116, "metrics/particle/class_3/pid_accuracy": 0.98145991563797, "metrics/particle/class_4/efficiency": 0.9074221849441528, "metrics/particle/class_4/pid_accuracy": 0.5180299282073975, "metrics/particle/class_5/efficiency": 0.9783549904823303, "metrics/particle/class_5/pid_accuracy": 0.7315634489059448, "metrics/particle/count/bias_mean": -1.4628571271896362, "metrics/particle/count/mae": 4.426285743713379, "metrics/particle/count/prediction_mean": 90.6914291381836, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04265028238296509, "metrics/particle/event/energy_response_mean": 0.9954646229743958, "metrics/particle/event/met_abs_error": 4.650439739227295, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03300875052809715, "metrics/particle/event/scalar_pt_response_mean": 0.9954462647438049, "metrics/particle/event/vector_pt_closure": 0.03476448729634285, "metrics/particle/matched/delta_eta_abs_mean": 0.011073770932853222, "metrics/particle/matched/delta_phi_abs_mean": 0.01168668083846569, "metrics/particle/matched/delta_r_mean": 0.01805146224796772, "metrics/particle/matched/energy_relative_abs_mean": 0.08793634921312332, "metrics/particle/matched/pid_accuracy": 0.9622090458869934, "metrics/particle/matched/pt_relative_abs_mean": 0.08243359625339508, "metrics/particle/matching/duplicate_fraction": 0.03045806847512722, "metrics/particle/matching/efficiency": 0.7386928796768188, "metrics/particle/matching/f1": 0.7446027994155884, "metrics/particle/matching/purity": 0.7506080269813538, "metrics/particle/matching/target_coverage_dr0p05": 0.6939666271209717, "metrics/particle/matching/target_coverage_dr0p10": 0.771612823009491, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6635828018188477, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8153903484344482}, "step_train_time": 17.31939935684204, "step_valid_time": 13.635920864995569, "step_total_time": 53628.61745214462, "elapsed_training_loop_seconds": 53628.61745214462, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018749125159958, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06640344364518047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1425, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03835800807537012, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9589502018842531, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9589502018842531, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9589502018842531, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9616419919246298, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9985567827824863, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07575720492479032, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1082, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1078, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9677996422182469, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9634906500445236, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.965640339134315, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0365093499554764, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.964221824686941, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9599287622439893, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.962070504239179, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9677996422182469, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026024596660705, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07181197699597619, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 583, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 582, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.928343949044586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9268680445151033, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9276054097056484, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07313195548489666, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9267515923566879, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9252782193958664, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9260143198090692, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.928343949044586, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000769223662311, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.049128012856333836, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1498, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1436, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1433, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9624664879356568, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9586114819759679, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9605351170568562, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04138851802403204, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960455764075067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9566088117489987, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9585284280936455, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9624664879356568, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9978225971621252, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0665544037607485, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1100, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1053, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1049, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9477947794779478, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9572727272727273, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9525101763907734, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042727272727272725, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9441944194419442, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9536363636363636, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9488919041157847, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9477947794779478, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020123600152906, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06276834657390018, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 591, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 589, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07075471698113207, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9261006289308176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9261006289308176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9261006289308176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9292452830188679, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9920627651855236, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11956755119404094, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3599, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2714, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2686, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.765153650972653, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7540983606557377, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7595857822558074, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2459016393442623, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.757259656047364, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7463184217838288, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.751749230338651, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.765153650972653, "step_test_time": 38.33700151799712, "step_plot_time": 26.375299755018204, "step_validation_cycle_time": 79.25222308700904} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_420000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_420000.json new file mode 100644 index 0000000000000000000000000000000000000000..aa1c8cfde347bddcb73e044a87c0ccc62855d844 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_420000.json @@ -0,0 +1 @@ +{"epoch": 1.0659492658908925, "train": {"Classification_binary": 2.197695732116699, "Classification": 0.02027554251253605, "Regression_pt": 0.23311972618103027, "Regression_eta": 0.06397861987352371, "Regression_sin_phi": 0.06297579407691956, "Regression_cos_phi": 0.05526549369096756, "Regression_energy": 0.2683335542678833, "Total": 8.326740264892578, "Total_unweighted": 2.9016444627195597}, "train_group": {"cld": {"Classification_binary": 2.181464957403349, "Classification": 0.015278662999605274, "Regression_pt": 0.12099211557849392, "Regression_eta": 0.012942376760514202, "Regression_sin_phi": 0.007038027861969428, "Regression_cos_phi": 0.007158753542892366, "Regression_energy": 0.1265494980298363, "Total": 4.654572070897879, "Total_unweighted": 2.4714243921766608}, "clic": {"Classification_binary": 2.113201527266173, "Classification": 0.016459440854419995, "Regression_pt": 0.10955834694139592, "Regression_eta": 0.010132400923780538, "Regression_sin_phi": 0.008606193813022685, "Regression_cos_phi": 0.008430188675564868, "Regression_energy": 0.11053900778651596, "Total": 4.402898368176755, "Total_unweighted": 2.376927106260873}, "colliderml": {"Classification_binary": 2.2294154324217472, "Classification": 0.02011077455663217, "Regression_pt": 0.2637067951574297, "Regression_eta": 0.057785335415137744, "Regression_sin_phi": 0.06388966114578133, "Regression_cos_phi": 0.06398451250202641, "Regression_energy": 0.30009554214998635, "Total": 8.943523908089734, "Total_unweighted": 2.9989880533487407}}, "valid": {"Classification_binary": 2.1704625909978694, "Classification": 0.016528954289176247, "Regression_pt": 0.1394576592878862, "Regression_eta": 0.017524175210432574, "Regression_sin_phi": 0.015262455290014094, "Regression_cos_phi": 0.015395545959472656, "Regression_energy": 0.14684202020818538, "Total": 5.2006464177911935, "group/cld/Classification": 0.014991511689855698, "group/cld/Classification_binary": 2.1717599097718585, "group/cld/Regression_cos_phi": 0.007014775529820869, "group/cld/Regression_energy": 0.12619154504004945, "group/cld/Regression_eta": 0.011779820665400079, "group/cld/Regression_pt": 0.12163933287275598, "group/cld/Regression_sin_phi": 0.006552963180744902, "group/cld/Total": 4.630541537670379, "group/cld/Total_unweighted": 2.4599298587504856, "group/clic/Classification": 0.01693761982816331, "group/clic/Classification_binary": 2.151952215965758, "group/clic/Regression_cos_phi": 0.00793982503261972, "group/clic/Regression_energy": 0.11890818210358316, "group/clic/Regression_eta": 0.010273667726110904, "group/clic/Regression_pt": 0.11627281473038044, "group/clic/Regression_sin_phi": 0.007804246658974505, "group/clic/Total": 4.5541086399808846, "group/clic/Total_unweighted": 2.4300885720455905, "group/colliderml/Classification": 0.019844725728034973, "group/colliderml/Classification_binary": 2.221022129058838, "group/colliderml/Regression_cos_phi": 0.06191521883010864, "group/colliderml/Regression_energy": 0.28955864906311035, "group/colliderml/Regression_eta": 0.05569660663604736, "group/colliderml/Regression_pt": 0.2599044442176819, "group/colliderml/Regression_sin_phi": 0.06275507062673569, "group/colliderml/Total": 8.774535179138184, "group/colliderml/Total_unweighted": 2.970696844160557, "Total_unweighted": 2.5214734012430364, "metrics/particle/class_1/efficiency": 0.965729832649231, "metrics/particle/class_1/pid_accuracy": 0.9844647645950317, "metrics/particle/class_2/efficiency": 0.3037870228290558, "metrics/particle/class_2/pid_accuracy": 0.7325232028961182, "metrics/particle/class_3/efficiency": 0.6705366373062134, "metrics/particle/class_3/pid_accuracy": 0.9798520803451538, "metrics/particle/class_4/efficiency": 0.9114125967025757, "metrics/particle/class_4/pid_accuracy": 0.5157617926597595, "metrics/particle/class_5/efficiency": 0.9769119620323181, "metrics/particle/class_5/pid_accuracy": 0.7223042845726013, "metrics/particle/count/bias_mean": -1.0742857456207275, "metrics/particle/count/mae": 4.326857089996338, "metrics/particle/count/prediction_mean": 91.08000183105469, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04362115636467934, "metrics/particle/event/energy_response_mean": 1.0066980123519897, "metrics/particle/event/met_abs_error": 4.802661418914795, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03339739516377449, "metrics/particle/event/scalar_pt_response_mean": 1.0073480606079102, "metrics/particle/event/vector_pt_closure": 0.03607138991355896, "metrics/particle/matched/delta_eta_abs_mean": 0.01090147439390421, "metrics/particle/matched/delta_phi_abs_mean": 0.012213175185024738, "metrics/particle/matched/delta_r_mean": 0.01830139569938183, "metrics/particle/matched/energy_relative_abs_mean": 0.08959975838661194, "metrics/particle/matched/pid_accuracy": 0.961864709854126, "metrics/particle/matched/pt_relative_abs_mean": 0.08276708424091339, "metrics/particle/matching/duplicate_fraction": 0.0331137478351593, "metrics/particle/matching/efficiency": 0.7398276329040527, "metrics/particle/matching/f1": 0.7441651821136475, "metrics/particle/matching/purity": 0.7485538721084595, "metrics/particle/matching/target_coverage_dr0p05": 0.6964903473854065, "metrics/particle/matching/target_coverage_dr0p10": 0.7738016843795776, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6627705097198486, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8150368928909302}, "step_train_time": 15.07317328453064, "step_valid_time": 13.618566986988299, "step_total_time": 55004.166236400604, "elapsed_training_loop_seconds": 55004.166236400604, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0034150379577353, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06444591267576416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1434, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9637096774193549, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9643577673167452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.036290322580645164, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9603494623655914, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.960995292535306, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9650067294751009, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030207626188719, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07372853714666805, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1120, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1074, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9651162790697675, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9633928571428572, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9642537980339589, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03660714285714286, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960644007155635, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9589285714285715, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9597855227882037, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9651162790697675, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0059701818770914, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0727686961884465, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 582, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 582, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9267515923566879, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9252782193958664, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9260143198090692, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07472178060413355, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9267515923566879, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9252782193958664, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9260143198090692, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9267515923566879, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028148612562964, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.04858012668188927, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1494, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1436, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1433, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9624664879356568, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9611780455153949, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9618218352310783, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038821954484605084, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960455764075067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9591700133868809, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9598124581379772, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9624664879356568, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040581575541825, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06432168128076332, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1105, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1055, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1053, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9495949594959496, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547511312217195, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9521660649819494, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04524886877828054, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9477947794779478, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9529411764705882, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9503610108303249, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9495949594959496, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0063045443906264, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06420742294234338, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 638, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 594, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 592, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9339622641509434, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9310344827586207, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9324960753532182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06896551724137931, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9308176100628931, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9278996865203761, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9293563579277865, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9339622641509434, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9983633041250075, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11927167706723096, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3591, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2713, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2679, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.764871722582464, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7554998607630187, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7601569066965537, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24450013923698133, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7552861573160418, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.746031746031746, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7506304286915102, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.764871722582464, "step_test_time": 41.077535075019114, "step_plot_time": 27.17382412403822, "step_validation_cycle_time": 82.60463015595451} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_430000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_430000.json new file mode 100644 index 0000000000000000000000000000000000000000..37a4d161d609efdeda9543c4621ca014622de7c3 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_430000.json @@ -0,0 +1 @@ +{"epoch": 1.091329010316866, "train": {"Classification_binary": 2.284978151321411, "Classification": 0.018130309879779816, "Regression_pt": 0.1532660275697708, "Regression_eta": 0.007235864643007517, "Regression_sin_phi": 0.006551124155521393, "Regression_cos_phi": 0.009698419831693172, "Regression_energy": 0.15082025527954102, "Total": 5.264026165008545, "Total_unweighted": 2.630680152680725}, "train_group": {"cld": {"Classification_binary": 2.1631912631902868, "Classification": 0.0149553102276617, "Regression_pt": 0.11689913197013432, "Regression_eta": 0.012988618016661239, "Regression_sin_phi": 0.007178287404522434, "Regression_cos_phi": 0.006969115465410991, "Regression_energy": 0.12170271396993877, "Total": 4.5596751517164495, "Total_unweighted": 2.443884440244616}, "clic": {"Classification_binary": 2.1179185031770587, "Classification": 0.016510131592686113, "Regression_pt": 0.11238276164184462, "Regression_eta": 0.009911299920529016, "Regression_sin_phi": 0.008280723117286334, "Regression_cos_phi": 0.008136722801795771, "Regression_energy": 0.11297508286642241, "Total": 4.4432753678914665, "Total_unweighted": 2.386115225117623}, "colliderml": {"Classification_binary": 2.230199318390351, "Classification": 0.01999355313387719, "Regression_pt": 0.261213576784721, "Regression_eta": 0.05719536033881319, "Regression_sin_phi": 0.06358869025880869, "Regression_cos_phi": 0.06372479493687819, "Regression_energy": 0.29762052012993406, "Total": 8.89173919660551, "Total_unweighted": 2.993535813973383}}, "valid": {"Classification_binary": 2.162530378861861, "Classification": 0.016436259313063188, "Regression_pt": 0.1397049296985973, "Regression_eta": 0.017436288703571667, "Regression_sin_phi": 0.015176437117836693, "Regression_cos_phi": 0.015352094173431396, "Regression_energy": 0.14697145982222123, "Total": 5.192657470703125, "group/cld/Classification": 0.014922829384499408, "group/cld/Classification_binary": 2.1532531088971076, "group/cld/Regression_cos_phi": 0.007104585779474136, "group/cld/Regression_energy": 0.12578083606476478, "group/cld/Regression_eta": 0.011652397348525677, "group/cld/Regression_pt": 0.1214001939651814, "group/cld/Regression_sin_phi": 0.006548461127788462, "group/cld/Total": 4.606332007874834, "group/cld/Total_unweighted": 2.4406624125673417, "group/clic/Classification": 0.01679779113607204, "group/clic/Classification_binary": 2.153300833194814, "group/clic/Regression_cos_phi": 0.007792758180740032, "group/clic/Regression_energy": 0.11970222757217731, "group/clic/Regression_eta": 0.010166337515445465, "group/clic/Regression_pt": 0.11713294780000727, "group/clic/Regression_sin_phi": 0.0076639176683222995, "group/clic/Total": 4.5631295062126, "group/clic/Total_unweighted": 2.432556813067578, "group/colliderml/Classification": 0.01981993205845356, "group/colliderml/Classification_binary": 2.2168965339660645, "group/colliderml/Regression_cos_phi": 0.06178472191095352, "group/colliderml/Regression_energy": 0.2893220782279968, "group/colliderml/Regression_eta": 0.05578191205859184, "group/colliderml/Regression_pt": 0.2597804665565491, "group/colliderml/Regression_sin_phi": 0.062589131295681, "group/colliderml/Total": 8.764225006103516, "group/colliderml/Total_unweighted": 2.9659747760742903, "Total_unweighted": 2.5136078476905825, "metrics/particle/class_1/efficiency": 0.9650191068649292, "metrics/particle/class_1/pid_accuracy": 0.9849816560745239, "metrics/particle/class_2/efficiency": 0.3060453534126282, "metrics/particle/class_2/pid_accuracy": 0.7435788512229919, "metrics/particle/class_3/efficiency": 0.6764882802963257, "metrics/particle/class_3/pid_accuracy": 0.9794304370880127, "metrics/particle/class_4/efficiency": 0.9058260321617126, "metrics/particle/class_4/pid_accuracy": 0.5471365451812744, "metrics/particle/class_5/efficiency": 0.9797979593276978, "metrics/particle/class_5/pid_accuracy": 0.7349042892456055, "metrics/particle/count/bias_mean": -1.2697142362594604, "metrics/particle/count/mae": 4.343999862670898, "metrics/particle/count/prediction_mean": 90.88457489013672, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04176272451877594, "metrics/particle/event/energy_response_mean": 0.997532069683075, "metrics/particle/event/met_abs_error": 4.659433364868164, "metrics/particle/event/scalar_pt_relative_abs_error": 0.032744400203228, "metrics/particle/event/scalar_pt_response_mean": 1.0016462802886963, "metrics/particle/event/vector_pt_closure": 0.034801170229911804, "metrics/particle/matched/delta_eta_abs_mean": 0.010567576624453068, "metrics/particle/matched/delta_phi_abs_mean": 0.011444982141256332, "metrics/particle/matched/delta_r_mean": 0.017453277483582497, "metrics/particle/matched/energy_relative_abs_mean": 0.08803962171077728, "metrics/particle/matched/pid_accuracy": 0.9629610776901245, "metrics/particle/matched/pt_relative_abs_mean": 0.08219597488641739, "metrics/particle/matching/duplicate_fraction": 0.03106609359383583, "metrics/particle/matching/efficiency": 0.7424753308296204, "metrics/particle/matching/f1": 0.7476257681846619, "metrics/particle/matching/purity": 0.752848207950592, "metrics/particle/matching/target_coverage_dr0p05": 0.7004526853561401, "metrics/particle/matching/target_coverage_dr0p10": 0.7746450304985046, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6669684648513794, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8182240724563599}, "step_train_time": 13.06146240234375, "step_valid_time": 13.5288937400328, "step_total_time": 56366.935678720474, "elapsed_training_loop_seconds": 56366.935678720474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004560459831195, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06466755050415707, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1493, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9670255720053835, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9624916275954454, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9647532729103726, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03750837240455459, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9656796769851952, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9611520428667113, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9634105404498153, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9670255720053835, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025342856763748, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0697453379252787, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1120, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1083, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1080, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9686940966010733, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9669642857142857, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.967828418230563, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.033035714285714286, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9660107334525939, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9642857142857143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9651474530831099, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9686940966010733, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0067543611452785, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07458830775226133, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 632, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 587, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9347133757961783, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9287974683544303, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9317460317460318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07120253164556962, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9331210191082803, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9272151898734177, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9301587301587302, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9347133757961783, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002542014782312, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.04948145627975542, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1494, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1439, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1436, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9644772117962467, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.963186077643909, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9638312123241795, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03681392235609103, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9624664879356568, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9611780455153949, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9618218352310783, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9644772117962467, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009994484504696, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06497414591372797, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1102, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1051, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1049, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.945994599459946, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9537205081669692, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9498418436511523, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04627949183303085, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9441944194419442, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9519056261343013, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9480343425214641, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.945994599459946, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040558300794318, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06546526989473356, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 597, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 595, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9386792452830188, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9401574803149606, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.939417781274587, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05984251968503937, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9355345911949685, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.937007874015748, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9362706530291109, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9386792452830188, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9914992015375399, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1182889905895097, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3565, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2711, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2684, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7643078658020863, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7604488078541375, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7623734533183352, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.23955119214586254, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7566957992669862, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.752875175315568, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7547806524184477, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7643078658020863, "step_test_time": 38.448640601942316, "step_plot_time": 26.530341945006512, "step_validation_cycle_time": 79.40621116501279} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_440000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_440000.json new file mode 100644 index 0000000000000000000000000000000000000000..db72c0eac48c74f5547213041f7702f71669c7bf --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_440000.json @@ -0,0 +1 @@ +{"epoch": 1.1167087547428398, "train": {"Classification_binary": 2.337130308151245, "Classification": 0.016151001676917076, "Regression_pt": 0.11456331610679626, "Regression_eta": 0.010909903794527054, "Regression_sin_phi": 0.00765825854614377, "Regression_cos_phi": 0.00979813002049923, "Regression_energy": 0.11595930904150009, "Total": 4.716581344604492, "Total_unweighted": 2.6121702273376286}, "train_group": {"cld": {"Classification_binary": 2.176366221439373, "Classification": 0.015113557149116342, "Regression_pt": 0.11997743092991926, "Regression_eta": 0.013025012623110378, "Regression_sin_phi": 0.006921493869230106, "Regression_cos_phi": 0.006709228710945438, "Regression_energy": 0.12443009870233121, "Total": 4.617124706417233, "Total_unweighted": 2.462543043424026}, "clic": {"Classification_binary": 2.092968970001815, "Classification": 0.016119823818970584, "Regression_pt": 0.10933230130258434, "Regression_eta": 0.009562860316914801, "Regression_sin_phi": 0.008301061818593677, "Regression_cos_phi": 0.008288041641796287, "Regression_energy": 0.11074216592454625, "Total": 4.368385896711293, "Total_unweighted": 2.3553152248252207}, "colliderml": {"Classification_binary": 2.2278361234579, "Classification": 0.020038067930945765, "Regression_pt": 0.2652135669021635, "Regression_eta": 0.05731395440729889, "Regression_sin_phi": 0.06377688412730759, "Regression_cos_phi": 0.06371544716549707, "Regression_energy": 0.3015292835486186, "Total": 8.95782487456863, "Total_unweighted": 2.9994233275397315}}, "valid": {"Classification_binary": 2.152829950506037, "Classification": 0.016375437649813564, "Regression_pt": 0.13847525336525657, "Regression_eta": 0.017396136847409337, "Regression_sin_phi": 0.015182902596213601, "Regression_cos_phi": 0.015308138457211582, "Regression_energy": 0.14592919783158737, "Total": 5.162523859197443, "group/cld/Classification": 0.014798378690760186, "group/cld/Classification_binary": 2.1432030860413898, "group/cld/Regression_cos_phi": 0.007003532445177119, "group/cld/Regression_energy": 0.12401591970565472, "group/cld/Regression_eta": 0.01166787806977617, "group/cld/Regression_pt": 0.1194399468442227, "group/cld/Regression_sin_phi": 0.006490092962346179, "group/cld/Total": 4.56192243860123, "group/cld/Total_unweighted": 2.426618834759327, "group/clic/Classification": 0.01679079456532255, "group/clic/Classification_binary": 2.141229345443401, "group/clic/Regression_cos_phi": 0.007827139281212016, "group/clic/Regression_energy": 0.11897627850796313, "group/clic/Regression_eta": 0.010066704547151606, "group/clic/Regression_pt": 0.11623932452912027, "group/clic/Regression_sin_phi": 0.007749616465669997, "group/clic/Total": 4.538286249688331, "group/clic/Total_unweighted": 2.418879203339841, "group/colliderml/Classification": 0.01978795789182186, "group/colliderml/Classification_binary": 2.215181589126587, "group/colliderml/Regression_cos_phi": 0.06167837232351303, "group/colliderml/Regression_energy": 0.28947359323501587, "group/colliderml/Regression_eta": 0.05575311556458473, "group/colliderml/Regression_pt": 0.25970935821533203, "group/colliderml/Regression_sin_phi": 0.06255330890417099, "group/colliderml/Total": 8.760490417480469, "group/colliderml/Total_unweighted": 2.9641372952610254, "Total_unweighted": 2.501497017253529, "metrics/particle/class_1/efficiency": 0.965173602104187, "metrics/particle/class_1/pid_accuracy": 0.9839755296707153, "metrics/particle/class_2/efficiency": 0.30691391229629517, "metrics/particle/class_2/pid_accuracy": 0.7393519282341003, "metrics/particle/class_3/efficiency": 0.6698939800262451, "metrics/particle/class_3/pid_accuracy": 0.9798744320869446, "metrics/particle/class_4/efficiency": 0.9122107028961182, "metrics/particle/class_4/pid_accuracy": 0.5529308915138245, "metrics/particle/class_5/efficiency": 0.9783549904823303, "metrics/particle/class_5/pid_accuracy": 0.7241888046264648, "metrics/particle/count/bias_mean": -1.8960000276565552, "metrics/particle/count/mae": 4.510857105255127, "metrics/particle/count/prediction_mean": 90.25828552246094, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.040855057537555695, "metrics/particle/event/energy_response_mean": 1.0042915344238281, "metrics/particle/event/met_abs_error": 4.677846908569336, "metrics/particle/event/scalar_pt_relative_abs_error": 0.031775183975696564, "metrics/particle/event/scalar_pt_response_mean": 1.006169080734253, "metrics/particle/event/vector_pt_closure": 0.03438476845622063, "metrics/particle/matched/delta_eta_abs_mean": 0.010376045480370522, "metrics/particle/matched/delta_phi_abs_mean": 0.011154445819556713, "metrics/particle/matched/delta_r_mean": 0.01706690341234207, "metrics/particle/matched/energy_relative_abs_mean": 0.0880514532327652, "metrics/particle/matched/pid_accuracy": 0.962230920791626, "metrics/particle/matched/pt_relative_abs_mean": 0.08241798728704453, "metrics/particle/matching/duplicate_fraction": 0.031756483018398285, "metrics/particle/matching/efficiency": 0.7397779822349548, "metrics/particle/matching/f1": 0.7474672794342041, "metrics/particle/matching/purity": 0.7553180456161499, "metrics/particle/matching/target_coverage_dr0p05": 0.7027159333229065, "metrics/particle/matching/target_coverage_dr0p10": 0.7739877104759216, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.662479043006897, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8132076859474182}, "step_train_time": 16.89073133468628, "step_valid_time": 13.411820192006417, "step_total_time": 57713.493436813354, "elapsed_training_loop_seconds": 57713.493436813354, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036410585477367, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06121711119747586, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1489, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9623908663532572, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9633613445378151, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03760913364674278, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9629878869448183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9610476830087307, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9620168067226891, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9643337819650067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003894273585165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06746104740526004, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1082, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1077, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9677996422182469, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9634906500445236, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.965640339134315, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0365093499554764, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9633273703041145, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9590382902938558, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9611780455153949, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9677996422182469, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0063558660135614, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07407276695833764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9315286624203821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9337589784517158, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.064, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9315286624203821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9337589784517158, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9315286624203821, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019227910815218, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.04900037355225928, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1498, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1439, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1435, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9644772117962467, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9606141522029372, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9625418060200669, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03938584779706275, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9617962466487936, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9579439252336449, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.959866220735786, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9644772117962467, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031553873307864, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06490220985791528, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1100, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1050, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1049, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9450945094509451, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9545454545454546, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9497964721845319, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045454545454545456, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9441944194419442, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9536363636363636, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9488919041157847, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9450945094509451, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0083713018612304, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06450112944474684, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 597, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 593, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9386792452830188, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9386792452830188, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9386792452830188, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06132075471698113, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9386792452830188, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9971149238740433, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11854684100231705, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3584, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2691, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2663, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7586692979983084, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7508370535714286, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7547328565418595, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24916294642857142, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7507753030730194, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7430245535714286, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7468798205020334, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7586692979983084, "step_test_time": 38.83988611900713, "step_plot_time": 26.296773037989624, "step_validation_cycle_time": 79.44840855896473} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_450000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_450000.json new file mode 100644 index 0000000000000000000000000000000000000000..95ae00c9b5f0cff34eaa86cb25f4caf1db4bb86d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_450000.json @@ -0,0 +1 @@ +{"epoch": 1.1420884991688134, "train": {"Classification_binary": 2.2159273624420166, "Classification": 0.021253852173686028, "Regression_pt": 0.2516903877258301, "Regression_eta": 0.058435648679733276, "Regression_sin_phi": 0.060702428221702576, "Regression_cos_phi": 0.06959636509418488, "Regression_energy": 0.2961551249027252, "Total": 8.832672119140625, "Total_unweighted": 2.9737611692398787}, "train_group": {"cld": {"Classification_binary": 2.145626608673875, "Classification": 0.015127030855699166, "Regression_pt": 0.11627428419328667, "Regression_eta": 0.012543214420024474, "Regression_sin_phi": 0.006718422543165182, "Regression_cos_phi": 0.006804211152572039, "Regression_energy": 0.12085281278904494, "Total": 4.522510934520412, "Total_unweighted": 2.423946584627667}, "clic": {"Classification_binary": 2.0925716247644512, "Classification": 0.016153069011665678, "Regression_pt": 0.10974359182310892, "Regression_eta": 0.009748139383002848, "Regression_sin_phi": 0.008140879547553466, "Regression_cos_phi": 0.008383706818242465, "Regression_energy": 0.11110308873760807, "Total": 4.375111896354515, "Total_unweighted": 2.3558441000856325}, "colliderml": {"Classification_binary": 2.224915984861865, "Classification": 0.019958035540794897, "Regression_pt": 0.2592096180169882, "Regression_eta": 0.05721665055541221, "Regression_sin_phi": 0.06367495136257417, "Regression_cos_phi": 0.06394856655133699, "Regression_energy": 0.2946337695250254, "Total": 8.85057092712311, "Total_unweighted": 2.9835575764139968}}, "valid": {"Classification_binary": 2.1485891168767757, "Classification": 0.01632670814340765, "Regression_pt": 0.13749298615889116, "Regression_eta": 0.017356646060943604, "Regression_sin_phi": 0.015125980160453103, "Regression_cos_phi": 0.015259502150795676, "Regression_energy": 0.14482688036831942, "Total": 5.139238947088068, "group/cld/Classification": 0.01478606842933817, "group/cld/Classification_binary": 2.1349102588410074, "group/cld/Regression_cos_phi": 0.006989032664197556, "group/cld/Regression_energy": 0.12346612646224651, "group/cld/Regression_eta": 0.011616336538436566, "group/cld/Regression_pt": 0.11904515611364487, "group/cld/Regression_sin_phi": 0.006501800202308817, "group/cld/Total": 4.5456507256690495, "group/cld/Total_unweighted": 2.41731477925118, "group/clic/Classification": 0.016711294651031494, "group/clic/Classification_binary": 2.14065827714636, "group/clic/Regression_cos_phi": 0.007723963007013848, "group/clic/Regression_energy": 0.11761584180466672, "group/clic/Regression_eta": 0.010057363738404943, "group/clic/Regression_pt": 0.11491163740766809, "group/clic/Regression_sin_phi": 0.00764293112653367, "group/clic/Total": 4.51208041576629, "group/clic/Total_unweighted": 2.415321308881679, "group/colliderml/Classification": 0.019722629338502884, "group/colliderml/Classification_binary": 2.212067127227783, "group/colliderml/Regression_cos_phi": 0.06168960779905319, "group/colliderml/Regression_energy": 0.2875065207481384, "group/colliderml/Regression_eta": 0.0556604228913784, "group/colliderml/Regression_pt": 0.25801628828048706, "group/colliderml/Regression_sin_phi": 0.06244097650051117, "group/colliderml/Total": 8.725172996520996, "group/colliderml/Total_unweighted": 2.9571035727858543, "Total_unweighted": 2.4949778199195864, "metrics/particle/class_1/efficiency": 0.9661315679550171, "metrics/particle/class_1/pid_accuracy": 0.9840394258499146, "metrics/particle/class_2/efficiency": 0.3085207939147949, "metrics/particle/class_2/pid_accuracy": 0.7469031810760498, "metrics/particle/class_3/efficiency": 0.6727300882339478, "metrics/particle/class_3/pid_accuracy": 0.979211688041687, "metrics/particle/class_4/efficiency": 0.9249800443649292, "metrics/particle/class_4/pid_accuracy": 0.5366695523262024, "metrics/particle/class_5/efficiency": 0.9783549904823303, "metrics/particle/class_5/pid_accuracy": 0.744837760925293, "metrics/particle/count/bias_mean": -1.268571376800537, "metrics/particle/count/mae": 4.394285678863525, "metrics/particle/count/prediction_mean": 90.88571166992188, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.041401855647563934, "metrics/particle/event/energy_response_mean": 0.9989967346191406, "metrics/particle/event/met_abs_error": 4.715595245361328, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03249264508485794, "metrics/particle/event/scalar_pt_response_mean": 1.0042661428451538, "metrics/particle/event/vector_pt_closure": 0.03498217090964317, "metrics/particle/matched/delta_eta_abs_mean": 0.010340915992856026, "metrics/particle/matched/delta_phi_abs_mean": 0.01109370868653059, "metrics/particle/matched/delta_r_mean": 0.017003338783979416, "metrics/particle/matched/energy_relative_abs_mean": 0.0871378555893898, "metrics/particle/matched/pid_accuracy": 0.962323009967804, "metrics/particle/matched/pt_relative_abs_mean": 0.08213532716035843, "metrics/particle/matching/duplicate_fraction": 0.03241119161248207, "metrics/particle/matching/efficiency": 0.7417498826980591, "metrics/particle/matching/f1": 0.7468906044960022, "metrics/particle/matching/purity": 0.7521030902862549, "metrics/particle/matching/target_coverage_dr0p05": 0.702759325504303, "metrics/particle/matching/target_coverage_dr0p10": 0.7749984264373779, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6669188141822815, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8172567486763}, "step_train_time": 15.573097944259644, "step_valid_time": 12.64235967793502, "step_total_time": 59079.58107614517, "elapsed_training_loop_seconds": 59079.58107614517, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018702397381118, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0647884571760492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1430, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9636852723604573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9640094180961991, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0363147276395427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9623149394347241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9616677874915938, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9619912546249579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9643337819650067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002643028757741, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06714121818870322, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1072, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9588550983899821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9545859305431879, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9567157518964748, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04541406945681211, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9543828264758497, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.95013357079252, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9522534582775547, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9588550983899821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0050616000701387, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07191300561089176, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 591, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 588, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9410828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9395866454689984, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9403341288782816, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06041335453100159, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9363057324840764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9348171701112877, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9355608591885441, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9410828025477707, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9998051940860666, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.04842624301069676, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1434, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1430, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9611260053619303, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9604822505023443, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9608040201005025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03951774949765573, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9584450402144772, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9578030810448761, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9581239530988275, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9611260053619303, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9996593434361043, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06558249031749175, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1102, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1046, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1045, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9414941494149415, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9491833030852994, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9453230908269318, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050816696914700546, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9405940594059405, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9482758620689655, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9444193402620876, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9414941494149415, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028261160430987, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05943538115114655, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 591, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 588, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07075471698113207, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9245283018867925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9245283018867925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9245283018867925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9292452830188679, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9952141515130981, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1205099269082539, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3573, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2693, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2670, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7592331547786862, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7537083683179401, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7564606741573033, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2462916316820599, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7527488018043417, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7472712006717045, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.75, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7592331547786862, "step_test_time": 42.27025029796641, "step_plot_time": 26.39159001992084, "step_validation_cycle_time": 82.2009707839461} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_460000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_460000.json new file mode 100644 index 0000000000000000000000000000000000000000..d50cf4cf4a946d43aae70d11fdb131460e7b4de4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_460000.json @@ -0,0 +1 @@ +{"epoch": 1.167468243594787, "train": {"Classification_binary": 2.194716453552246, "Classification": 0.01359559502452612, "Regression_pt": 0.11054187268018723, "Regression_eta": 0.008415276184678078, "Regression_sin_phi": 0.007411843631416559, "Regression_cos_phi": 0.00430253567174077, "Regression_energy": 0.1090029701590538, "Total": 4.371565818786621, "Total_unweighted": 2.4479865469038486}, "train_group": {"cld": {"Classification_binary": 2.152173547687645, "Classification": 0.014965129374877779, "Regression_pt": 0.1161359145478931, "Regression_eta": 0.012955115375410163, "Regression_sin_phi": 0.006926979382332794, "Regression_cos_phi": 0.007050970864822407, "Regression_energy": 0.12106032007022532, "Total": 4.53476565398142, "Total_unweighted": 2.4312679773032064}, "clic": {"Classification_binary": 2.077878494162459, "Classification": 0.015910344222118636, "Regression_pt": 0.10913568339100829, "Regression_eta": 0.009753847477241635, "Regression_sin_phi": 0.007822978975916947, "Regression_cos_phi": 0.0077962470933090194, "Regression_energy": 0.11006104650798144, "Total": 4.333497336676887, "Total_unweighted": 2.3383586418300353}, "colliderml": {"Classification_binary": 2.225247864966636, "Classification": 0.019981924567494664, "Regression_pt": 0.2631271318630413, "Regression_eta": 0.05679446378269711, "Regression_sin_phi": 0.06353487490533709, "Regression_cos_phi": 0.06328602824304197, "Regression_energy": 0.29863345350052145, "Total": 8.903885520614303, "Total_unweighted": 2.9906057418287695}}, "valid": {"Classification_binary": 2.14392353404652, "Classification": 0.016266473856839266, "Regression_pt": 0.13726130398837003, "Regression_eta": 0.017339353127913043, "Regression_sin_phi": 0.015101429549130527, "Regression_cos_phi": 0.015265342322262851, "Regression_energy": 0.14452498175881126, "Total": 5.129168701171875, "group/cld/Classification": 0.014695948742805644, "group/cld/Classification_binary": 2.1315942317881484, "group/cld/Regression_cos_phi": 0.006960662755560368, "group/cld/Regression_energy": 0.12269204728146817, "group/cld/Regression_eta": 0.011655728867713442, "group/cld/Regression_pt": 0.11827238569868372, "group/cld/Regression_sin_phi": 0.006455496904697824, "group/cld/Total": 4.527770671438663, "group/cld/Total_unweighted": 2.412326502039078, "group/clic/Classification": 0.016681690165337097, "group/clic/Classification_binary": 2.1341981279089097, "group/clic/Regression_cos_phi": 0.007806410180761459, "group/clic/Regression_energy": 0.11773936291958423, "group/clic/Regression_eta": 0.010016098301461402, "group/clic/Regression_pt": 0.11514480063255797, "group/clic/Regression_sin_phi": 0.0076628866347860786, "group/clic/Total": 4.509307536673038, "group/clic/Total_unweighted": 2.409249376743398, "group/colliderml/Classification": 0.019660206511616707, "group/colliderml/Classification_binary": 2.2087085247039795, "group/colliderml/Regression_cos_phi": 0.06157097965478897, "group/colliderml/Regression_energy": 0.2873419523239136, "group/colliderml/Regression_eta": 0.055547084659338, "group/colliderml/Regression_pt": 0.25800856947898865, "group/colliderml/Regression_sin_phi": 0.062349602580070496, "group/colliderml/Total": 8.716625213623047, "group/colliderml/Total_unweighted": 2.953186919912696, "Total_unweighted": 2.489682418649847, "metrics/particle/class_1/efficiency": 0.965312659740448, "metrics/particle/class_1/pid_accuracy": 0.9842499494552612, "metrics/particle/class_2/efficiency": 0.30704420804977417, "metrics/particle/class_2/pid_accuracy": 0.7391796112060547, "metrics/particle/class_3/efficiency": 0.6717939972877502, "metrics/particle/class_3/pid_accuracy": 0.9799521565437317, "metrics/particle/class_4/efficiency": 0.9058260321617126, "metrics/particle/class_4/pid_accuracy": 0.5638766288757324, "metrics/particle/class_5/efficiency": 0.9754689931869507, "metrics/particle/class_5/pid_accuracy": 0.7366864085197449, "metrics/particle/count/bias_mean": -1.3600000143051147, "metrics/particle/count/mae": 4.32457160949707, "metrics/particle/count/prediction_mean": 90.7942886352539, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04139622673392296, "metrics/particle/event/energy_response_mean": 1.0032694339752197, "metrics/particle/event/met_abs_error": 4.598389625549316, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03222926706075668, "metrics/particle/event/scalar_pt_response_mean": 1.00631582736969, "metrics/particle/event/vector_pt_closure": 0.034131258726119995, "metrics/particle/matched/delta_eta_abs_mean": 0.010134258307516575, "metrics/particle/matched/delta_phi_abs_mean": 0.011003254912793636, "metrics/particle/matched/delta_r_mean": 0.016770541667938232, "metrics/particle/matched/energy_relative_abs_mean": 0.08754178881645203, "metrics/particle/matched/pid_accuracy": 0.962617814540863, "metrics/particle/matched/pt_relative_abs_mean": 0.08227044343948364, "metrics/particle/matching/duplicate_fraction": 0.031430549919605255, "metrics/particle/matching/efficiency": 0.7406337261199951, "metrics/particle/matching/f1": 0.746139407157898, "metrics/particle/matching/purity": 0.7517275810241699, "metrics/particle/matching/target_coverage_dr0p05": 0.7042971253395081, "metrics/particle/matching/target_coverage_dr0p10": 0.7744341492652893, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6647919416427612, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8154771327972412}, "step_train_time": 14.208934307098389, "step_valid_time": 12.709189358050935, "step_total_time": 60428.67301559448, "elapsed_training_loop_seconds": 60428.67301559448, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043425322015165, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06164334769821522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1491, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1438, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1434, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9676985195154778, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9644533869885983, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9660732280819617, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.035546613011401745, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9617706237424547, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9633859590191468, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9676985195154778, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.006229342022126, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06966588051896394, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1124, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1081, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1078, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9669051878354203, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9617437722419929, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9643175735950045, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03825622775800712, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.964221824686941, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9590747330960854, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9616413916146298, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9669051878354203, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007378733926722, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06877974870224124, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 587, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9347133757961783, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9376996805111821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9362041467304625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06230031948881789, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9331210191082803, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9361022364217252, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9346092503987241, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9347133757961783, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0027497336403897, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.04906286022119166, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1498, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1445, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1442, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9684986595174263, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9646194926568759, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9665551839464883, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.035380507343124167, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9664879356568364, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9626168224299065, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9645484949832775, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9684986595174263, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0044661701011435, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06535811014973159, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1058, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1056, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9522952295229523, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9592021758839528, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.955736224028907, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04079782411604715, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9504950495049505, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.957388939256573, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9539295392953929, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9522952295229523, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0087304124782277, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06151524073707315, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 593, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 590, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06761006289308176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9323899371069182, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9960480530731113, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11978807861732566, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3583, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2700, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2674, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7612066535100085, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7535584705554005, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7573632538569425, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2464415294445995, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7538765153650973, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7463019815796819, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7500701262272089, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7612066535100085, "step_test_time": 39.98831363895442, "step_plot_time": 26.300763707025908, "step_validation_cycle_time": 79.9086752210278} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_470000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_470000.json new file mode 100644 index 0000000000000000000000000000000000000000..d0f3ceaf7f78702dff2ee0e70baf610c1b55e993 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_470000.json @@ -0,0 +1 @@ +{"epoch": 1.1928479880207605, "train": {"Classification_binary": 2.0490546226501465, "Classification": 0.013896970078349113, "Regression_pt": 0.08305055648088455, "Regression_eta": 0.0037288351450115442, "Regression_sin_phi": 0.008310097269713879, "Regression_cos_phi": 0.006936544086784124, "Regression_energy": 0.08278112858533859, "Total": 3.802521228790283, "Total_unweighted": 2.2477587542962283}, "train_group": {"cld": {"Classification_binary": 2.1486931448584206, "Classification": 0.015312612146228642, "Regression_pt": 0.11627751749915045, "Regression_eta": 0.01143574206418618, "Regression_sin_phi": 0.006889181989091294, "Regression_cos_phi": 0.006975114933570271, "Regression_energy": 0.120917106399665, "Total": 4.52615337400465, "Total_unweighted": 2.426500419890312}, "clic": {"Classification_binary": 2.071988741437832, "Classification": 0.015953456464329524, "Regression_pt": 0.11040435215000978, "Regression_eta": 0.00918203166234279, "Regression_sin_phi": 0.008032284544580087, "Regression_cos_phi": 0.007996540184796392, "Regression_energy": 0.11133237207631864, "Total": 4.350582589646299, "Total_unweighted": 2.334889778520209}, "colliderml": {"Classification_binary": 2.218716234774203, "Classification": 0.020033748904327013, "Regression_pt": 0.2611599965048982, "Regression_eta": 0.05731318953695956, "Regression_sin_phi": 0.06338424766967604, "Regression_cos_phi": 0.06342135440256144, "Regression_energy": 0.2970257821652266, "Total": 8.87097404239414, "Total_unweighted": 2.981054553957852}}, "valid": {"Classification_binary": 2.1391985806551848, "Classification": 0.01624372655695135, "Regression_pt": 0.13649708140980113, "Regression_eta": 0.01726455580104481, "Regression_sin_phi": 0.015100689367814498, "Regression_cos_phi": 0.01526215293190696, "Regression_energy": 0.14377195184881036, "Total": 5.1111988414417615, "group/cld/Classification": 0.014660835266113281, "group/cld/Classification_binary": 2.1235025284138134, "group/cld/Regression_cos_phi": 0.006959162493969532, "group/cld/Regression_energy": 0.12196993320546251, "group/cld/Regression_eta": 0.011515300324622621, "group/cld/Regression_pt": 0.1176344891811939, "group/cld/Regression_sin_phi": 0.006478245588059121, "group/cld/Total": 4.507637510908411, "group/cld/Total_unweighted": 2.4027204944732348, "group/clic/Classification": 0.01666383920831883, "group/clic/Classification_binary": 2.1321198483730885, "group/clic/Regression_cos_phi": 0.007791647251616133, "group/clic/Regression_energy": 0.11712985343121468, "group/clic/Regression_eta": 0.009975816341156655, "group/clic/Regression_pt": 0.11446029581922165, "group/clic/Regression_sin_phi": 0.007630630376491141, "group/clic/Total": 4.4954519068941154, "group/clic/Total_unweighted": 2.405771930801108, "group/colliderml/Classification": 0.019659407436847687, "group/colliderml/Classification_binary": 2.20609974861145, "group/colliderml/Regression_cos_phi": 0.061596792191267014, "group/colliderml/Regression_energy": 0.28607672452926636, "group/colliderml/Regression_eta": 0.05556365102529526, "group/colliderml/Regression_pt": 0.25663939118385315, "group/colliderml/Regression_sin_phi": 0.06237243115901947, "group/colliderml/Total": 8.692930221557617, "group/colliderml/Total_unweighted": 2.948008146136999, "Total_unweighted": 2.4833387385715135, "metrics/particle/class_1/efficiency": 0.9656216502189636, "metrics/particle/class_1/pid_accuracy": 0.9842389822006226, "metrics/particle/class_2/efficiency": 0.3100842535495758, "metrics/particle/class_2/pid_accuracy": 0.7425770163536072, "metrics/particle/class_3/efficiency": 0.6736381649971008, "metrics/particle/class_3/pid_accuracy": 0.9794055819511414, "metrics/particle/class_4/efficiency": 0.916201114654541, "metrics/particle/class_4/pid_accuracy": 0.5583623647689819, "metrics/particle/class_5/efficiency": 0.9783549904823303, "metrics/particle/class_5/pid_accuracy": 0.7315634489059448, "metrics/particle/count/bias_mean": -1.2851428985595703, "metrics/particle/count/mae": 4.253142833709717, "metrics/particle/count/prediction_mean": 90.869140625, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.04109816625714302, "metrics/particle/event/energy_response_mean": 0.9999268054962158, "metrics/particle/event/met_abs_error": 4.6939697265625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.032061073929071426, "metrics/particle/event/scalar_pt_response_mean": 1.0036004781723022, "metrics/particle/event/vector_pt_closure": 0.034483253955841064, "metrics/particle/matched/delta_eta_abs_mean": 0.010052956640720367, "metrics/particle/matched/delta_phi_abs_mean": 0.010977798141539097, "metrics/particle/matched/delta_r_mean": 0.016698606312274933, "metrics/particle/matched/energy_relative_abs_mean": 0.08752278983592987, "metrics/particle/matched/pid_accuracy": 0.9623576402664185, "metrics/particle/matched/pt_relative_abs_mean": 0.08207666128873825, "metrics/particle/matching/duplicate_fraction": 0.031574446707963943, "metrics/particle/matching/efficiency": 0.7421032786369324, "metrics/particle/matching/f1": 0.7473141551017761, "metrics/particle/matching/purity": 0.7525987029075623, "metrics/particle/matching/target_coverage_dr0p05": 0.7042909264564514, "metrics/particle/matching/target_coverage_dr0p10": 0.7752279043197632, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6662057638168335, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8166056871414185}, "step_train_time": 14.209001541137695, "step_valid_time": 12.740819684928283, "step_total_time": 61790.00548005104, "elapsed_training_loop_seconds": 61790.00548005104, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0001952973958415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.060856423425924455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1438, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9676985195154778, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9670477471418965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9673730238816011, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03295225285810356, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9657027572293208, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9660275815674403, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9676985195154778, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018512913299473, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06748363644255395, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1124, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1083, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9686940966010733, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9635231316725978, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9661016949152542, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.036476868327402136, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9651162790697675, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9599644128113879, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9625334522747547, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9686940966010733, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0045882011813771, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06715677317225355, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 583, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 582, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.928343949044586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9328, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9305666400638468, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0672, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9267515923566879, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9312, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9289704708699122, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.928343949044586, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001239176699804, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.049034857115173525, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1447, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1444, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9698391420911529, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9665998663994656, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9682167949146872, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.033400133600534405, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.967828418230563, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9645958583834335, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9662094345935095, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9698391420911529, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002292890775709, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06301375819867183, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1058, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1056, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9522952295229523, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9592021758839528, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.955736224028907, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04079782411604715, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9504950495049505, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.957388939256573, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9539295392953929, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9522952295229523, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0056286643788048, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06220907891049954, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 593, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 590, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9338582677165355, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9331235247836349, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06614173228346457, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9291338582677166, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9284028324154209, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9323899371069182, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.995629537366742, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11826212673093961, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3593, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2705, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2679, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7626162954609529, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7528527692735876, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.757703081232493, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24714723072641248, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7552861573160418, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7456164764820484, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7504201680672269, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7626162954609529, "step_test_time": 40.92475310002919, "step_plot_time": 26.22349911299534, "step_validation_cycle_time": 80.76824407710228} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_480000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_480000.json new file mode 100644 index 0000000000000000000000000000000000000000..c4540e1ba975e7b7a48a18c7efd23e01c62d5e42 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_480000.json @@ -0,0 +1 @@ +{"epoch": 1.2182277324467343, "train": {"Classification_binary": 2.0889904499053955, "Classification": 0.01886373944580555, "Regression_pt": 0.26569420099258423, "Regression_eta": 0.0549163855612278, "Regression_sin_phi": 0.05933540686964989, "Regression_cos_phi": 0.06178874522447586, "Regression_energy": 0.3156033754348755, "Total": 8.80814266204834, "Total_unweighted": 2.8651923034340143}, "train_group": {"cld": {"Classification_binary": 2.1234667537448644, "Classification": 0.01483077816553302, "Regression_pt": 0.11474405161969296, "Regression_eta": 0.013236964655992528, "Regression_sin_phi": 0.0067654112591645915, "Regression_cos_phi": 0.0069412297447556046, "Regression_energy": 0.11987521799208524, "Total": 4.481152454296032, "Total_unweighted": 2.3998604071820884}, "clic": {"Classification_binary": 2.0587848786477214, "Classification": 0.016161280519164004, "Regression_pt": 0.10880330699431645, "Regression_eta": 0.009352938497897697, "Regression_sin_phi": 0.00807739454097822, "Regression_cos_phi": 0.008099706596680903, "Regression_energy": 0.10975025717292104, "Total": 4.3168616624207825, "Total_unweighted": 2.3190297629696794}, "colliderml": {"Classification_binary": 2.216628012542953, "Classification": 0.019934502262956726, "Regression_pt": 0.2587803174367922, "Regression_eta": 0.05734331795421546, "Regression_sin_phi": 0.06362882748380036, "Regression_cos_phi": 0.06333750447336428, "Regression_energy": 0.29439364913516414, "Total": 8.82908087147924, "Total_unweighted": 2.974046131289246}}, "valid": {"Classification_binary": 2.1353959517045453, "Classification": 0.0162023511799899, "Regression_pt": 0.13634512641213156, "Regression_eta": 0.0172832727432251, "Regression_sin_phi": 0.015097410028631037, "Regression_cos_phi": 0.015235046906904741, "Regression_energy": 0.14361502907492898, "Total": 5.104062721946023, "group/cld/Classification": 0.01461627001458026, "group/cld/Classification_binary": 2.120181956189744, "group/cld/Regression_cos_phi": 0.00694760743607866, "group/cld/Regression_energy": 0.12157590338524352, "group/cld/Regression_eta": 0.011542069151046429, "group/cld/Regression_pt": 0.11719236982629654, "group/cld/Regression_sin_phi": 0.006467006942059131, "group/cld/Total": 4.496664818297041, "group/cld/Total_unweighted": 2.3985231829450484, "group/clic/Classification": 0.016612615991146006, "group/clic/Classification_binary": 2.1276464259370846, "group/clic/Regression_cos_phi": 0.0077723981218135105, "group/clic/Regression_energy": 0.11712569378791972, "group/clic/Regression_eta": 0.009986782327611396, "group/clic/Regression_pt": 0.1144947904221555, "group/clic/Regression_sin_phi": 0.0076336366065005035, "group/clic/Total": 4.490471373213098, "group/clic/Total_unweighted": 2.401272343194231, "group/colliderml/Classification": 0.019656330347061157, "group/colliderml/Classification_binary": 2.202848196029663, "group/colliderml/Regression_cos_phi": 0.0615009106695652, "group/colliderml/Regression_energy": 0.2861673831939697, "group/colliderml/Regression_eta": 0.05558149516582489, "group/colliderml/Regression_pt": 0.25679144263267517, "group/colliderml/Regression_sin_phi": 0.062374047935009, "group/colliderml/Total": 8.690713882446289, "group/colliderml/Total_unweighted": 2.9449198059737682, "Total_unweighted": 2.4791741880503566, "metrics/particle/class_1/efficiency": 0.9655289649963379, "metrics/particle/class_1/pid_accuracy": 0.9843975305557251, "metrics/particle/class_2/efficiency": 0.3113871216773987, "metrics/particle/class_2/pid_accuracy": 0.7415620684623718, "metrics/particle/class_3/efficiency": 0.6762787103652954, "metrics/particle/class_3/pid_accuracy": 0.9790108799934387, "metrics/particle/class_4/efficiency": 0.9114125967025757, "metrics/particle/class_4/pid_accuracy": 0.5446584820747375, "metrics/particle/class_5/efficiency": 0.9740259647369385, "metrics/particle/class_5/pid_accuracy": 0.7303703427314758, "metrics/particle/count/bias_mean": -0.9651428461074829, "metrics/particle/count/mae": 4.192571640014648, "metrics/particle/count/prediction_mean": 91.18914031982422, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.040005940943956375, "metrics/particle/event/energy_response_mean": 1.001800298690796, "metrics/particle/event/met_abs_error": 4.5762763023376465, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03176343813538551, "metrics/particle/event/scalar_pt_response_mean": 1.0044317245483398, "metrics/particle/event/vector_pt_closure": 0.033743344247341156, "metrics/particle/matched/delta_eta_abs_mean": 0.009967546910047531, "metrics/particle/matched/delta_phi_abs_mean": 0.010907243005931377, "metrics/particle/matched/delta_r_mean": 0.01657017692923546, "metrics/particle/matched/energy_relative_abs_mean": 0.08770179003477097, "metrics/particle/matched/pid_accuracy": 0.9620797038078308, "metrics/particle/matched/pt_relative_abs_mean": 0.08221550285816193, "metrics/particle/matching/duplicate_fraction": 0.03125058859586716, "metrics/particle/matching/efficiency": 0.7433682680130005, "metrics/particle/matching/f1": 0.7472814321517944, "metrics/particle/matching/purity": 0.7512360215187073, "metrics/particle/matching/target_coverage_dr0p05": 0.7058101296424866, "metrics/particle/matching/target_coverage_dr0p10": 0.776095986366272, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.667247474193573, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8187573552131653}, "step_train_time": 13.032312870025635, "step_valid_time": 12.70310637599323, "step_total_time": 63139.861347436905, "elapsed_training_loop_seconds": 63139.861347436905, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0024288143175244, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06080833650362172, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03566621803499327, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9643337819650067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031692954674367, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06845161695189117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1124, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1083, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9686940966010733, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9635231316725978, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9661016949152542, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.036476868327402136, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9651162790697675, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9599644128113879, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9625334522747547, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9686940966010733, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005327134232584, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07095909790865296, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 591, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 589, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9410828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9410828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9410828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0589171974522293, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9378980891719745, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9378980891719745, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9378980891719745, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9410828025477707, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013868085418498, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.04833681752066887, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1495, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1448, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1444, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9705093833780161, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.968561872909699, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9695346501506529, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.031438127090301006, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.967828418230563, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9658862876254181, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9668563776364245, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9705093833780161, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0021526912468262, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.062314197181260966, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1057, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1054, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9513951395139514, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.958295557570263, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.95483288166215, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04170444242973708, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9486948694869487, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9555757026291931, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.952122854561879, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9513951395139514, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004460924820531, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06236556555148287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 593, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 590, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9323899371069182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06761006289308176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9276729559748428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9323899371069182, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9957821373877268, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11820388586215214, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3583, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2694, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2670, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7595150831688751, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7518838961763885, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.755680224403927, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2481161038236115, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7527488018043417, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7451855986603405, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7489481065918654, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7595150831688751, "step_test_time": 38.66807487991173, "step_plot_time": 27.32866976701189, "step_validation_cycle_time": 79.59614383801818} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_490000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_490000.json new file mode 100644 index 0000000000000000000000000000000000000000..cac5754169fd916475610369d7d0312cf14968a7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_490000.json @@ -0,0 +1 @@ +{"epoch": 1.243607476872708, "train": {"Classification_binary": 1.977490782737732, "Classification": 0.015446294099092484, "Regression_pt": 0.1485152393579483, "Regression_eta": 0.002441232791170478, "Regression_sin_phi": 0.005729714408516884, "Regression_cos_phi": 0.005162706132978201, "Regression_energy": 0.1494864821434021, "Total": 4.772491455078125, "Total_unweighted": 2.3042724516708404}, "train_group": {"cld": {"Classification_binary": 2.130447657165413, "Classification": 0.01497779616773753, "Regression_pt": 0.1173316304332125, "Regression_eta": 0.011817930493514962, "Regression_sin_phi": 0.006989685940545075, "Regression_cos_phi": 0.0069116433680860585, "Regression_energy": 0.1218037424709447, "Total": 4.522284322156164, "Total_unweighted": 2.410280086039454}, "clic": {"Classification_binary": 2.073812209450089, "Classification": 0.0162464290264401, "Regression_pt": 0.10741229387777704, "Regression_eta": 0.00945217184954555, "Regression_sin_phi": 0.008226578188892286, "Regression_cos_phi": 0.008074906254718612, "Regression_energy": 0.10839087742047983, "Total": 4.312570684306972, "Total_unweighted": 2.3316154660679422}, "colliderml": {"Classification_binary": 2.2142591268808633, "Classification": 0.0199327653738203, "Regression_pt": 0.25929724901646106, "Regression_eta": 0.05724842235580221, "Regression_sin_phi": 0.06358579597673616, "Regression_cos_phi": 0.06379964235084909, "Regression_energy": 0.29490521729171454, "Total": 8.839530523832854, "Total_unweighted": 2.9730282192462467}}, "valid": {"Classification_binary": 2.1344797307794745, "Classification": 0.01620014255697077, "Regression_pt": 0.13620967864990235, "Regression_eta": 0.017274738441814076, "Regression_sin_phi": 0.01506972529671409, "Regression_cos_phi": 0.01523031538183039, "Regression_energy": 0.14352107481522994, "Total": 5.100721324573864, "group/cld/Classification": 0.014617261734414608, "group/cld/Classification_binary": 2.118992419953042, "group/cld/Regression_cos_phi": 0.006936680763325793, "group/cld/Regression_energy": 0.12141600060970226, "group/cld/Regression_eta": 0.011534662956887104, "group/cld/Regression_pt": 0.11697551037402863, "group/cld/Regression_sin_phi": 0.0064571290574175245, "group/cld/Total": 4.492011536943152, "group/cld/Total_unweighted": 2.396929665448818, "group/clic/Classification": 0.016614609576286153, "group/clic/Classification_binary": 2.126528963129571, "group/clic/Regression_cos_phi": 0.007762592523655993, "group/clic/Regression_energy": 0.11705432039626101, "group/clic/Regression_eta": 0.00998546024586292, "group/clic/Regression_pt": 0.11441337301376019, "group/clic/Regression_sin_phi": 0.007603885011470064, "group/clic/Total": 4.4876228494847075, "group/clic/Total_unweighted": 2.3999632038968675, "group/colliderml/Classification": 0.0196323674172163, "group/colliderml/Classification_binary": 2.2033307552337646, "group/colliderml/Regression_cos_phi": 0.06152929365634918, "group/colliderml/Regression_energy": 0.28620126843452454, "group/colliderml/Regression_eta": 0.05554845929145813, "group/colliderml/Regression_pt": 0.25673672556877136, "group/colliderml/Regression_sin_phi": 0.062300123274326324, "group/colliderml/Total": 8.689788818359375, "group/colliderml/Total_unweighted": 2.9452789928764105, "Total_unweighted": 2.477985405921936, "metrics/particle/class_1/efficiency": 0.9652817249298096, "metrics/particle/class_1/pid_accuracy": 0.9849057197570801, "metrics/particle/class_2/efficiency": 0.30934596061706543, "metrics/particle/class_2/pid_accuracy": 0.7442088723182678, "metrics/particle/class_3/efficiency": 0.674797773361206, "metrics/particle/class_3/pid_accuracy": 0.9789233803749084, "metrics/particle/class_4/efficiency": 0.9130087494850159, "metrics/particle/class_4/pid_accuracy": 0.5428321957588196, "metrics/particle/class_5/efficiency": 0.9783549904823303, "metrics/particle/class_5/pid_accuracy": 0.7271386384963989, "metrics/particle/count/bias_mean": -1.287428617477417, "metrics/particle/count/mae": 4.25428581237793, "metrics/particle/count/prediction_mean": 90.86685943603516, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.040468085557222366, "metrics/particle/event/energy_response_mean": 0.9995830655097961, "metrics/particle/event/met_abs_error": 4.571784973144531, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0316624715924263, "metrics/particle/event/scalar_pt_response_mean": 1.0028940439224243, "metrics/particle/event/vector_pt_closure": 0.03376508876681328, "metrics/particle/matched/delta_eta_abs_mean": 0.009974034503102303, "metrics/particle/matched/delta_phi_abs_mean": 0.01087844930589199, "metrics/particle/matched/delta_r_mean": 0.01655280962586403, "metrics/particle/matched/energy_relative_abs_mean": 0.08718906342983246, "metrics/particle/matched/pid_accuracy": 0.9624871611595154, "metrics/particle/matched/pt_relative_abs_mean": 0.08187291026115417, "metrics/particle/matching/duplicate_fraction": 0.031009262427687645, "metrics/particle/matching/efficiency": 0.7423513531684875, "metrics/particle/matching/f1": 0.7475732564926147, "metrics/particle/matching/purity": 0.752869188785553, "metrics/particle/matching/target_coverage_dr0p05": 0.7050722241401672, "metrics/particle/matching/target_coverage_dr0p10": 0.7747752070426941, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6667762398719788, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8180876970291138}, "step_train_time": 11.78972339630127, "step_valid_time": 12.86001074500382, "step_total_time": 64502.164638757706, "elapsed_training_loop_seconds": 64502.164638757706, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015402140171104, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.061128619669391315, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1434, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.033647375504710635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9663526244952894, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032744237898783, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06846917919730844, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1083, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9686940966010733, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9643811219946572, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9665327978580991, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03561887800534283, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9651162790697675, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9608192341941229, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9629629629629629, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9686940966010733, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0056975096342478, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0713843093873614, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 588, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 587, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9363057324840764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.937799043062201, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9370517928286852, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06220095693779904, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347133757961783, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9362041467304625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9354581673306773, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9363057324840764, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000879320098552, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.04916669851115363, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1494, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1448, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1445, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9705093833780161, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9692101740294511, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9698593436034829, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.030789825970548863, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9684986595174263, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9672021419009371, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9678499665103818, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9705093833780161, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0014128317853739, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06372542491109079, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1060, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1057, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9540954095409541, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9610154125113327, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.957542908762421, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038984587488667274, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9513951395139514, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.958295557570263, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.95483288166215, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9540954095409541, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003943366857353, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06627347552361229, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 591, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 588, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9292452830188679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07075471698113207, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9245283018867925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9245283018867925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9245283018867925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9292452830188679, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9943866815699269, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11864520489116259, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3580, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2700, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2669, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7612066535100085, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7541899441340782, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7576820541602357, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24581005586592178, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7524668734141529, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7455307262569832, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7489827416865441, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7612066535100085, "step_test_time": 38.60234549304005, "step_plot_time": 27.310099345981143, "step_validation_cycle_time": 79.66504271095619} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..f63e4f00206683dde0e3a95f16214f71979f1d72 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 0.12689872212986816, "train": {"Classification_binary": 3.639967203140259, "Classification": 0.0227513387799263, "Regression_pt": 0.2598438262939453, "Regression_eta": 0.011180277913808823, "Regression_sin_phi": 0.007570106070488691, "Regression_cos_phi": 0.009618915617465973, "Regression_energy": 0.2574310302734375, "Total": 8.462982177734375, "Total_unweighted": 4.208362698089331}, "train_group": {"cld": {"Classification_binary": 3.668773810784738, "Classification": 0.027329758094908955, "Regression_pt": 0.22787403993241423, "Regression_eta": 0.02244072607033201, "Regression_sin_phi": 0.011582633288851282, "Regression_cos_phi": 0.011547679833888828, "Regression_energy": 0.23597521905426508, "Total": 8.236712929722783, "Total_unweighted": 4.205523867059398}, "clic": {"Classification_binary": 3.3781430949707945, "Classification": 0.029326031263360006, "Regression_pt": 0.18888778752552535, "Regression_eta": 0.016023362738576657, "Regression_sin_phi": 0.013031037927265027, "Regression_cos_phi": 0.013054276802604962, "Regression_energy": 0.19128131886531494, "Total": 7.294949488725491, "Total_unweighted": 3.8297469100934416}, "colliderml": {"Classification_binary": 2.9109162718684107, "Classification": 0.0289033121141317, "Regression_pt": 0.3739642457919078, "Regression_eta": 0.06468981079332105, "Regression_sin_phi": 0.07270789622992009, "Regression_cos_phi": 0.0730748964143587, "Regression_energy": 0.41914772146098966, "Total": 11.864851504832775, "Total_unweighted": 3.94340415467304}}, "valid": {"Classification_binary": 3.404151777787642, "Classification": 0.02862957607616078, "Regression_pt": 0.2359140396118164, "Regression_eta": 0.02546122507615523, "Regression_sin_phi": 0.020174349438060412, "Regression_cos_phi": 0.020642885294827547, "Regression_energy": 0.2473956194790927, "Total": 8.37464766068892, "group/cld/Classification": 0.026806681714159376, "group/cld/Classification_binary": 3.621287893741689, "group/cld/Regression_cos_phi": 0.011689387737436498, "group/cld/Regression_energy": 0.24078892646951877, "group/cld/Regression_eta": 0.02133247192869795, "group/cld/Regression_pt": 0.2316382184941718, "group/cld/Regression_sin_phi": 0.010479097036605186, "group/cld/Total": 8.232312790890957, "group/cld/Total_unweighted": 4.164022677122278, "group/clic/Classification": 0.030473983034174493, "group/clic/Classification_binary": 3.360860621675532, "group/clic/Regression_cos_phi": 0.012489105792755776, "group/clic/Regression_energy": 0.19685410438699924, "group/clic/Regression_eta": 0.01685907358818866, "group/clic/Regression_pt": 0.19249816651039936, "group/clic/Regression_sin_phi": 0.01235673909491681, "group/clic/Total": 7.355085656998005, "group/clic/Total_unweighted": 3.8223917940829666, "group/colliderml/Classification": 0.028566328808665276, "group/colliderml/Classification_binary": 2.8934807777404785, "group/colliderml/Regression_cos_phi": 0.07089550793170929, "group/colliderml/Regression_energy": 0.4152687191963196, "group/colliderml/Regression_eta": 0.06285825371742249, "group/colliderml/Regression_pt": 0.376008540391922, "group/colliderml/Regression_sin_phi": 0.07161840796470642, "group/colliderml/Total": 11.787715911865234, "group/colliderml/Total_unweighted": 3.9186965357512236, "Total_unweighted": 3.982369472763755, "metrics/particle/class_1/efficiency": 0.929250180721283, "metrics/particle/class_1/pid_accuracy": 0.9760400652885437, "metrics/particle/class_2/efficiency": 0.17267437279224396, "metrics/particle/class_2/pid_accuracy": 0.38707244396209717, "metrics/particle/class_3/efficiency": 0.5645668506622314, "metrics/particle/class_3/pid_accuracy": 0.9652808904647827, "metrics/particle/class_4/efficiency": 0.8858739137649536, "metrics/particle/class_4/pid_accuracy": 0.11081080883741379, "metrics/particle/class_5/efficiency": 0.9653679728507996, "metrics/particle/class_5/pid_accuracy": 0.28550073504447937, "metrics/particle/count/bias_mean": -1.9165713787078857, "metrics/particle/count/mae": 5.843428611755371, "metrics/particle/count/prediction_mean": 90.23771667480469, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.07657681405544281, "metrics/particle/event/energy_response_mean": 0.9854764938354492, "metrics/particle/event/met_abs_error": 9.294438362121582, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06353791058063507, "metrics/particle/event/scalar_pt_response_mean": 0.9960184097290039, "metrics/particle/event/vector_pt_closure": 0.06788934767246246, "metrics/particle/matched/delta_eta_abs_mean": 0.023229192942380905, "metrics/particle/matched/delta_phi_abs_mean": 0.01843632012605667, "metrics/particle/matched/delta_r_mean": 0.03295092657208443, "metrics/particle/matched/energy_relative_abs_mean": 0.093111053109169, "metrics/particle/matched/pid_accuracy": 0.936542272567749, "metrics/particle/matched/pt_relative_abs_mean": 0.08936718851327896, "metrics/particle/matching/duplicate_fraction": 0.043580129742622375, "metrics/particle/matching/efficiency": 0.6591864824295044, "metrics/particle/matching/f1": 0.6661131978034973, "metrics/particle/matching/purity": 0.6731870174407959, "metrics/particle/matching/target_coverage_dr0p05": 0.532566487789154, "metrics/particle/matching/target_coverage_dr0p10": 0.6928567290306091, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6029143929481506, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7599801421165466}, "step_train_time": 7.317703723907471, "step_valid_time": 12.360136567964219, "step_total_time": 6228.577007293701, "elapsed_training_loop_seconds": 6228.577007293701, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007896927801232, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14767001805085944, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1356, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1331, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9125168236877523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9137466307277629, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9131313131313131, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0862533692722372, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.895693135935397, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8969002695417789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8962962962962963, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9125168236877523, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9930373260954366, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1660358206336362, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1115, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1004, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 992, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8980322003577818, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9004484304932735, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8992386923421406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09955156950672646, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8872987477638641, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8896860986547085, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8884908195253023, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8980322003577818, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0075643625539814, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1487606161483792, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 620, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 514, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 503, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8184713375796179, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8290322580645161, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8237179487179487, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17096774193548386, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8009554140127388, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8112903225806452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8060897435897436, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8184713375796179, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9936656310470436, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10919037689544098, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1378, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1371, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9235924932975871, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9205076820307281, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9220475075276012, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07949231796927188, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9189008042895442, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9158316633266533, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9173636667781867, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9235924932975871, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.990984576488926, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13097965369912845, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1097, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 984, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 975, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8856885688568857, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8969917958067457, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8913043478260869, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10300820419325434, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8775877587758776, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8887876025524157, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8831521739130435, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8856885688568857, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9929451710440333, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1371603718528821, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 637, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 534, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 522, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.839622641509434, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8383045525902669, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8389630793401414, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.16169544740973313, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8207547169811321, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8194662480376766, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8201099764336214, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.839622641509434, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9857763106772257, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24763397353145766, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3592, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2564, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2435, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7228643924443191, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7138084632516704, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7183078862585797, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.28619153674832964, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6864956301099521, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6778953229398663, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6821683709202969, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7228643924443191, "step_test_time": 33.600273068994284, "step_plot_time": 26.325419467990287, "step_validation_cycle_time": 72.79591686697677} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_500000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_500000.json new file mode 100644 index 0000000000000000000000000000000000000000..753162e3fea35a78417ad459b4d4f8ce3674d0e9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_500000.json @@ -0,0 +1 @@ +{"epoch": 1.2689872212986815, "train": {"Classification_binary": 2.205685615539551, "Classification": 0.019231026992201805, "Regression_pt": 0.14055393636226654, "Regression_eta": 0.007334961090236902, "Regression_sin_phi": 0.010026160627603531, "Regression_cos_phi": 0.005987596232444048, "Regression_energy": 0.14068497717380524, "Total": 5.007207870483398, "Total_unweighted": 2.529504274018109}, "train_group": {"cld": {"Classification_binary": 2.120731276434821, "Classification": 0.014925570934787169, "Regression_pt": 0.11565640628024622, "Regression_eta": 0.011979123364571851, "Regression_sin_phi": 0.0065724586602300406, "Regression_cos_phi": 0.006823196134938336, "Regression_energy": 0.12087269282869033, "Total": 4.484042148332338, "Total_unweighted": 2.397560724638285}, "clic": {"Classification_binary": 2.0671663791119697, "Classification": 0.016159496682802005, "Regression_pt": 0.11098471105455639, "Regression_eta": 0.009186580775774078, "Regression_sin_phi": 0.007902829795601795, "Regression_cos_phi": 0.008009620327335185, "Regression_energy": 0.1115973045837558, "Total": 4.3545969489091885, "Total_unweighted": 2.331006922331795}, "colliderml": {"Classification_binary": 2.212934394498487, "Classification": 0.01980801086660262, "Regression_pt": 0.2609881244085214, "Regression_eta": 0.05672284231350586, "Regression_sin_phi": 0.06340289743499714, "Regression_cos_phi": 0.06337329701663137, "Regression_energy": 0.2960873053328053, "Total": 8.850510439715228, "Total_unweighted": 2.973316871871551}}, "valid": {"Classification_binary": 2.1343917846679688, "Classification": 0.01618618965148926, "Regression_pt": 0.13600371100685812, "Regression_eta": 0.017265701293945314, "Regression_sin_phi": 0.015079976211894643, "Regression_cos_phi": 0.015233297781510786, "Regression_energy": 0.14329723011363638, "Total": 5.097061434659091, "group/cld/Classification": 0.01459747426053311, "group/cld/Classification_binary": 2.118674582623421, "group/cld/Regression_cos_phi": 0.0069379235835785565, "group/cld/Regression_energy": 0.12123281397718064, "group/cld/Regression_eta": 0.011518212074929096, "group/cld/Regression_pt": 0.11678734231502452, "group/cld/Regression_sin_phi": 0.006462393923008696, "group/cld/Total": 4.4883575439453125, "group/cld/Total_unweighted": 2.3962107427576766, "group/clic/Classification": 0.01660679502690092, "group/clic/Classification_binary": 2.12638465394365, "group/clic/Regression_cos_phi": 0.007765005877677431, "group/clic/Regression_energy": 0.11692954124288356, "group/clic/Regression_eta": 0.009977420593829864, "group/clic/Regression_pt": 0.11434513457277988, "group/clic/Regression_sin_phi": 0.007617519891008418, "group/clic/Total": 4.486033987491689, "group/clic/Total_unweighted": 2.39962607114873, "group/colliderml/Classification": 0.019617510959506035, "group/colliderml/Classification_binary": 2.2040820121765137, "group/colliderml/Regression_cos_phi": 0.06153906509280205, "group/colliderml/Regression_energy": 0.285566508769989, "group/colliderml/Regression_eta": 0.05555824190378189, "group/colliderml/Regression_pt": 0.2560741603374481, "group/colliderml/Regression_sin_phi": 0.062315069139003754, "group/colliderml/Total": 8.680020332336426, "group/colliderml/Total_unweighted": 2.9447525683790445, "Total_unweighted": 2.477457890727303, "metrics/particle/class_1/efficiency": 0.9652972221374512, "metrics/particle/class_1/pid_accuracy": 0.9844897985458374, "metrics/particle/class_2/efficiency": 0.3086945116519928, "metrics/particle/class_2/pid_accuracy": 0.7418401837348938, "metrics/particle/class_3/efficiency": 0.6741132140159607, "metrics/particle/class_3/pid_accuracy": 0.9792335629463196, "metrics/particle/class_4/efficiency": 0.9114125967025757, "metrics/particle/class_4/pid_accuracy": 0.542907178401947, "metrics/particle/class_5/efficiency": 0.9783549904823303, "metrics/particle/class_5/pid_accuracy": 0.7271386384963989, "metrics/particle/count/bias_mean": -1.194857120513916, "metrics/particle/count/mae": 4.24057149887085, "metrics/particle/count/prediction_mean": 90.95942687988281, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.040541671216487885, "metrics/particle/event/energy_response_mean": 1.001595377922058, "metrics/particle/event/met_abs_error": 4.607748508453369, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03182273730635643, "metrics/particle/event/scalar_pt_response_mean": 1.0042208433151245, "metrics/particle/event/vector_pt_closure": 0.03420122340321541, "metrics/particle/matched/delta_eta_abs_mean": 0.009936289861798286, "metrics/particle/matched/delta_phi_abs_mean": 0.01086423173546791, "metrics/particle/matched/delta_r_mean": 0.01651584915816784, "metrics/particle/matched/energy_relative_abs_mean": 0.08765031397342682, "metrics/particle/matched/pid_accuracy": 0.9622828960418701, "metrics/particle/matched/pt_relative_abs_mean": 0.08191332221031189, "metrics/particle/matching/duplicate_fraction": 0.031423743814229965, "metrics/particle/matching/efficiency": 0.7419483065605164, "metrics/particle/matching/f1": 0.7467896342277527, "metrics/particle/matching/purity": 0.7516946196556091, "metrics/particle/matching/target_coverage_dr0p05": 0.7051342725753784, "metrics/particle/matching/target_coverage_dr0p10": 0.7746574282646179, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6666831970214844, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8173807859420776}, "step_train_time": 15.297810554504395, "step_valid_time": 12.645378728979267, "step_total_time": 65847.84989857674, "elapsed_training_loop_seconds": 65847.84989857674, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0027151889119226, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.061215497917156636, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1434, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9643577673167452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9646821392532795, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03564223268325487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9629878869448183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9623402824478816, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9626639757820383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9650067294751009, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043291159320473, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07107220830086025, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1124, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1084, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1080, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9695885509838998, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9644128113879004, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9669937555753791, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03558718861209965, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9660107334525939, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9608540925266904, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9634255129348795, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9695885509838998, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0053903212079098, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07037516860863269, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 588, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9363057324840764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.937799043062201, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9370517928286852, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06220095693779904, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9331210191082803, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9346092503987241, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9338645418326693, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9363057324840764, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001897044203245, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.048025251472601393, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1449, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1446, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9711796246648794, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.97052913596785, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9708542713567839, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.029470864032150032, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9691689008042895, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9685197588747488, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9688442211055276, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9711796246648794, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000865637261803, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0640536721817817, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1102, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1058, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1055, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9522952295229523, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9600725952813067, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.956168097605061, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.039927404718693285, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9495949594959496, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.957350272232305, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534568459105287, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9522952295229523, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058523365459058, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06112811792870443, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 637, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 592, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 589, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9308176100628931, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9293563579277865, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9300864100549883, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0706436420722135, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9261006289308176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9246467817896389, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9253731343283582, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9308176100628931, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9960153965341454, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11483214918784534, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3583, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2694, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2658, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7595150831688751, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7518838961763885, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.755680224403927, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2481161038236115, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.749365661122075, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7418364499023165, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7455820476858345, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7595150831688751, "step_test_time": 38.860719389049336, "step_plot_time": 27.541063430020586, "step_validation_cycle_time": 79.94108339201193} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_60000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..d10aee381e8f75a4db27bf1838b2228a3c01b115 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 0.1522784665558418, "train": {"Classification_binary": 2.8940486907958984, "Classification": 0.027768706902861595, "Regression_pt": 0.327091783285141, "Regression_eta": 0.06474851071834564, "Regression_sin_phi": 0.07060201466083527, "Regression_cos_phi": 0.07279689610004425, "Regression_energy": 0.36673352122306824, "Total": 10.999626159667969, "Total_unweighted": 3.8237901236861944}, "train_group": {"cld": {"Classification_binary": 3.514182197439062, "Classification": 0.02567139496696783, "Regression_pt": 0.21577928521790662, "Regression_eta": 0.0212189656201269, "Regression_sin_phi": 0.010686704195342265, "Regression_cos_phi": 0.010642897413187736, "Regression_energy": 0.2236353481585557, "Total": 7.830968660635275, "Total_unweighted": 4.0218167930111495}, "clic": {"Classification_binary": 3.2711546542766214, "Classification": 0.0281608730230156, "Regression_pt": 0.18472854437323305, "Regression_eta": 0.015453365242651274, "Regression_sin_phi": 0.012605765342220172, "Regression_cos_phi": 0.012692462833312986, "Regression_energy": 0.18580420838819967, "Total": 7.082581535831944, "Total_unweighted": 3.7105998734792545}, "colliderml": {"Classification_binary": 2.865957797644381, "Classification": 0.0278359878986717, "Regression_pt": 0.3598013110681922, "Regression_eta": 0.06361903015992598, "Regression_sin_phi": 0.07138027825175289, "Regression_cos_phi": 0.07134531518656337, "Regression_energy": 0.4051824289703084, "Total": 11.531526448483952, "Total_unweighted": 3.8651221491797956}}, "valid": {"Classification_binary": 3.304971590909091, "Classification": 0.027317155491222034, "Regression_pt": 0.22545524943958628, "Regression_eta": 0.02410718094218861, "Regression_sin_phi": 0.01936604543165727, "Regression_cos_phi": 0.019893043691461738, "Regression_energy": 0.2353038094260476, "Total": 8.048894708806818, "group/cld/Classification": 0.025709433758512458, "group/cld/Classification_binary": 3.5068365868101727, "group/cld/Regression_cos_phi": 0.010966472169186206, "group/cld/Regression_energy": 0.22744085433635305, "group/cld/Regression_eta": 0.020109923596077776, "group/cld/Regression_pt": 0.22123340850180767, "group/cld/Regression_sin_phi": 0.009835081531646405, "group/cld/Total": 7.889649086810173, "group/cld/Total_unweighted": 4.022131760703757, "group/clic/Classification": 0.02887705792772009, "group/clic/Classification_binary": 3.2597873768907912, "group/clic/Regression_cos_phi": 0.011893320590891737, "group/clic/Regression_energy": 0.18770451241351188, "group/clic/Regression_eta": 0.015335691736099567, "group/clic/Regression_pt": 0.1833485948278549, "group/clic/Regression_sin_phi": 0.011544642296243222, "group/clic/Total": 7.056478297456782, "group/clic/Total_unweighted": 3.698491196683112, "group/colliderml/Classification": 0.02745767869055271, "group/colliderml/Classification_binary": 2.844721555709839, "group/colliderml/Regression_cos_phi": 0.06961402297019958, "group/colliderml/Regression_energy": 0.3982238471508026, "group/colliderml/Regression_eta": 0.06161535158753395, "group/colliderml/Regression_pt": 0.3615451157093048, "group/colliderml/Regression_sin_phi": 0.07033862173557281, "group/colliderml/Total": 11.431886672973633, "group/colliderml/Total_unweighted": 3.8335161935538054, "Total_unweighted": 3.856414075331255, "metrics/particle/class_1/efficiency": 0.9317223429679871, "metrics/particle/class_1/pid_accuracy": 0.9753905534744263, "metrics/particle/class_2/efficiency": 0.18687570095062256, "metrics/particle/class_2/pid_accuracy": 0.40878456830978394, "metrics/particle/class_3/efficiency": 0.5803260803222656, "metrics/particle/class_3/pid_accuracy": 0.9675959348678589, "metrics/particle/class_4/efficiency": 0.8635275363922119, "metrics/particle/class_4/pid_accuracy": 0.16266173124313354, "metrics/particle/class_5/efficiency": 0.9653679728507996, "metrics/particle/class_5/pid_accuracy": 0.33781763911247253, "metrics/particle/count/bias_mean": -0.838857114315033, "metrics/particle/count/mae": 5.305142879486084, "metrics/particle/count/prediction_mean": 91.3154296875, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.0738484337925911, "metrics/particle/event/energy_response_mean": 1.007298231124878, "metrics/particle/event/met_abs_error": 8.980987548828125, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05796413496136665, "metrics/particle/event/scalar_pt_response_mean": 0.9971827864646912, "metrics/particle/event/vector_pt_closure": 0.0656547024846077, "metrics/particle/matched/delta_eta_abs_mean": 0.02144516631960869, "metrics/particle/matched/delta_phi_abs_mean": 0.018264075741171837, "metrics/particle/matched/delta_r_mean": 0.031353339552879333, "metrics/particle/matched/energy_relative_abs_mean": 0.09654613584280014, "metrics/particle/matched/pid_accuracy": 0.9376888275146484, "metrics/particle/matched/pt_relative_abs_mean": 0.09276710450649261, "metrics/particle/matching/duplicate_fraction": 0.04361021891236305, "metrics/particle/matching/efficiency": 0.6690270900726318, "metrics/particle/matching/f1": 0.6720860004425049, "metrics/particle/matching/purity": 0.6751730442047119, "metrics/particle/matching/target_coverage_dr0p05": 0.5551807284355164, "metrics/particle/matching/target_coverage_dr0p10": 0.7033918499946594, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6091957688331604, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7684256434440613}, "step_train_time": 7.329190015792847, "step_valid_time": 12.343719205935486, "step_total_time": 7486.249956846237, "elapsed_training_loop_seconds": 7486.249956846237, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9933916337306363, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14206395251199622, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1363, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1347, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9172274562584118, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9209459459459459, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9190829399865138, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07905405405405405, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9064602960969045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9101351351351351, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9082939986513824, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9172274562584118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9882965977364971, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15473741536778352, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1113, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 991, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 979, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8864042933810375, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8903863432165319, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8883908561183326, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1096136567834681, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8756708407871199, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8796046720575023, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8776333482743165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8864042933810375, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00063816513029, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13881987136673246, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 619, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 513, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 509, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8168789808917197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8287560581583199, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8227746591820368, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17124394184168013, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8105095541401274, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8222940226171244, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8163592622293504, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8168789808917197, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9908217296570518, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10811499789444001, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1386, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1379, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9289544235924933, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9283322170127261, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.928643216080402, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07166778298727394, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9242627345844504, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9236436704621568, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9239530988274707, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9289544235924933, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9851970587125091, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12291102125739184, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1090, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 993, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 988, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8937893789378938, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9110091743119266, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9023171285779191, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0889908256880734, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8892889288928892, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9064220183486239, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8977737392094502, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8937893789378938, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9864225304798101, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12254172321593193, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 639, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 547, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 541, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.860062893081761, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8560250391236307, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8580392156862745, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.14397496087636932, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.85062893081761, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8466353677621283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8486274509803922, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.860062893081761, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9870806084031201, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2226680810263758, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3647, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2611, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2499, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7361150267831971, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7159309021113244, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7258826800111203, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2840690978886756, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7045390470820412, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6852207293666027, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6947456213511259, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7361150267831971, "step_test_time": 33.732299180934206, "step_plot_time": 26.475337099982426, "step_validation_cycle_time": 73.06500025291461} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_70000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..2740d5e734e4326784f025da0ec26c597f18a343 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 0.17765821098181542, "train": {"Classification_binary": 3.579071044921875, "Classification": 0.020532215014100075, "Regression_pt": 0.171319380402565, "Regression_eta": 0.023982100188732147, "Regression_sin_phi": 0.01620224118232727, "Regression_cos_phi": 0.01163056306540966, "Regression_energy": 0.17855311930179596, "Total": 7.189306259155273, "Total_unweighted": 4.001290664076805}, "train_group": {"cld": {"Classification_binary": 3.4282293398223236, "Classification": 0.025371357344068632, "Regression_pt": 0.2105258817667376, "Regression_eta": 0.02029157031890741, "Regression_sin_phi": 0.010672053457432967, "Regression_cos_phi": 0.010602271105553174, "Regression_energy": 0.21779636478798833, "Total": 7.644936455937916, "Total_unweighted": 3.923488838603012}, "clic": {"Classification_binary": 3.1627081084895776, "Classification": 0.027525123158419453, "Regression_pt": 0.1755556170870592, "Regression_eta": 0.013671181703059075, "Regression_sin_phi": 0.011967046344345127, "Regression_cos_phi": 0.011966624495771912, "Regression_energy": 0.1776881374127872, "Total": 6.795172624043874, "Total_unweighted": 3.58108183869102}, "colliderml": {"Classification_binary": 2.8104235159384237, "Classification": 0.026798379302651318, "Regression_pt": 0.3452988909529494, "Regression_eta": 0.0631187384774735, "Regression_sin_phi": 0.07108707467580701, "Regression_cos_phi": 0.07082446926348918, "Regression_energy": 0.3899806020138142, "Total": 11.207016437977284, "Total_unweighted": 3.7775316706246085}}, "valid": {"Classification_binary": 3.2204409512606533, "Classification": 0.0262774489142678, "Regression_pt": 0.2171245054765181, "Regression_eta": 0.02303478717803955, "Regression_sin_phi": 0.019110614603216, "Regression_cos_phi": 0.01921343369917436, "Regression_energy": 0.22619965293190697, "Total": 7.791225363991478, "group/cld/Classification": 0.024473961363447472, "group/cld/Classification_binary": 3.407565502410239, "group/cld/Regression_cos_phi": 0.010238789497537816, "group/cld/Regression_energy": 0.21647402580748212, "group/cld/Regression_eta": 0.018560224391044454, "group/cld/Regression_pt": 0.21013464826218625, "group/cld/Regression_sin_phi": 0.00965013973256375, "group/cld/Total": 7.572620797664561, "group/cld/Total_unweighted": 3.897097291464501, "group/clic/Classification": 0.028003555663088535, "group/clic/Classification_binary": 3.177794436190991, "group/clic/Regression_cos_phi": 0.011292548889809467, "group/clic/Regression_energy": 0.17908171389965302, "group/clic/Regression_eta": 0.014532124742548516, "group/clic/Regression_pt": 0.17623335249880526, "group/clic/Regression_sin_phi": 0.011454922087649082, "group/clic/Total": 6.823277899559508, "group/clic/Total_unweighted": 3.598392653972545, "group/colliderml/Classification": 0.02650477923452854, "group/colliderml/Classification_binary": 2.7960386276245117, "group/colliderml/Regression_cos_phi": 0.06884407997131348, "group/colliderml/Regression_energy": 0.3931775391101837, "group/colliderml/Regression_eta": 0.061155375093221664, "group/colliderml/Regression_pt": 0.35777515172958374, "group/colliderml/Regression_sin_phi": 0.06938936561346054, "group/colliderml/Total": 11.27672004699707, "group/colliderml/Total_unweighted": 3.7728849183768034, "Total_unweighted": 3.7514013940637763, "metrics/particle/class_1/efficiency": 0.9349979162216187, "metrics/particle/class_1/pid_accuracy": 0.9772779941558838, "metrics/particle/class_2/efficiency": 0.18978546559810638, "metrics/particle/class_2/pid_accuracy": 0.44073227047920227, "metrics/particle/class_3/efficiency": 0.5850343108177185, "metrics/particle/class_3/pid_accuracy": 0.9668298363685608, "metrics/particle/class_4/efficiency": 0.8770949840545654, "metrics/particle/class_4/pid_accuracy": 0.15923567116260529, "metrics/particle/class_5/efficiency": 0.9595959782600403, "metrics/particle/class_5/pid_accuracy": 0.39398497343063354, "metrics/particle/count/bias_mean": -0.6759999990463257, "metrics/particle/count/mae": 5.534285545349121, "metrics/particle/count/prediction_mean": 91.47828674316406, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.06568650901317596, "metrics/particle/event/energy_response_mean": 1.0007944107055664, "metrics/particle/event/met_abs_error": 8.493626594543457, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05466519668698311, "metrics/particle/event/scalar_pt_response_mean": 1.0015268325805664, "metrics/particle/event/vector_pt_closure": 0.0629713237285614, "metrics/particle/matched/delta_eta_abs_mean": 0.01991412043571472, "metrics/particle/matched/delta_phi_abs_mean": 0.017274197190999985, "metrics/particle/matched/delta_r_mean": 0.02938818372786045, "metrics/particle/matched/energy_relative_abs_mean": 0.09506665915250778, "metrics/particle/matched/pid_accuracy": 0.9397823214530945, "metrics/particle/matched/pt_relative_abs_mean": 0.08948719501495361, "metrics/particle/matching/duplicate_fraction": 0.0429329052567482, "metrics/particle/matching/efficiency": 0.6729273796081543, "metrics/particle/matching/f1": 0.6754046082496643, "metrics/particle/matching/purity": 0.6779001355171204, "metrics/particle/matching/target_coverage_dr0p05": 0.5620760321617126, "metrics/particle/matching/target_coverage_dr0p10": 0.7089663147926331, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6103305220603943, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7703726887702942}, "step_train_time": 7.329310655593872, "step_valid_time": 12.352801036904566, "step_total_time": 8756.800905942917, "elapsed_training_loop_seconds": 8756.800905942917, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9903619483031494, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13685763737300904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1467, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1376, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1364, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9259757738896366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9379686434901159, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9319336268201829, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06203135650988412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.917900403768506, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9297886843899114, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9238062986793092, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9259757738896366, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9904157342232438, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14087921885060772, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1012, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1002, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9051878354203936, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.90437890974084, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9047831917746982, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09562109025915996, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8962432915921288, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8954423592493298, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8958426464014305, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9051878354203936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0062575070070923, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14531613304722824, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 620, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 518, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 513, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8248407643312102, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8354838709677419, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8301282051282052, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.16451612903225807, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8168789808917197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8274193548387097, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8221153846153846, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8248407643312102, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9914653023934565, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09143068389042197, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1376, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1371, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9222520107238605, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9216342933690556, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9219430485762145, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0783657066309444, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9189008042895442, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9182853315472204, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9185929648241206, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9222520107238605, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9941238743189684, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11505784869449331, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1004, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1002, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9036903690369037, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9102447869446962, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9069557362240289, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08975521305530372, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9018901890189019, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9084315503173164, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9051490514905149, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9036903690369037, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9968733880500844, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11022956405333617, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 639, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 551, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 548, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8663522012578616, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8622848200312989, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.864313725490196, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13771517996870108, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8616352201257862, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8575899843505478, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8596078431372549, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8663522012578616, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9755411646423262, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20400961789540162, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3630, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2569, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2494, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7242740343952636, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7077134986225895, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7158980075240351, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.29228650137741047, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7031294051310967, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6870523415977962, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6949979099902466, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7242740343952636, "step_test_time": 33.79147530300543, "step_plot_time": 26.686576660955325, "step_validation_cycle_time": 73.33865787100513} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_80000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..3b1bf08bf9c7ce659a8cdc396aafa0546a29483e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 0.20303795540778904, "train": {"Classification_binary": 3.267158269882202, "Classification": 0.028595177456736565, "Regression_pt": 0.19324061274528503, "Regression_eta": 0.010821709409356117, "Regression_sin_phi": 0.011997183784842491, "Regression_cos_phi": 0.00782652385532856, "Regression_energy": 0.1935853660106659, "Total": 7.126669883728027, "Total_unweighted": 3.713224843144417}, "train_group": {"cld": {"Classification_binary": 3.34443897384781, "Classification": 0.024121225733231676, "Regression_pt": 0.20394293480628245, "Regression_eta": 0.01955978067272955, "Regression_sin_phi": 0.010055562994490425, "Regression_cos_phi": 0.010477841433204777, "Regression_energy": 0.21143987184172278, "Total": 7.424493126683049, "Total_unweighted": 3.824036191329472}, "clic": {"Classification_binary": 3.083631599020815, "Classification": 0.026366449122925004, "Regression_pt": 0.17228674342086214, "Regression_eta": 0.01396992460344946, "Regression_sin_phi": 0.01166521255512482, "Regression_cos_phi": 0.011617212242608567, "Regression_energy": 0.17421460923498977, "Total": 6.639420630689153, "Total_unweighted": 3.493751750200775}, "colliderml": {"Classification_binary": 2.762175307975517, "Classification": 0.02613402698356826, "Regression_pt": 0.349182002089761, "Regression_eta": 0.062432449736126196, "Regression_sin_phi": 0.07015490434593982, "Regression_cos_phi": 0.06992265740940878, "Regression_energy": 0.39385220954368066, "Total": 11.187156276301936, "Total_unweighted": 3.733853558084002}}, "valid": {"Classification_binary": 3.1216089422052558, "Classification": 0.025497505881569602, "Regression_pt": 0.21095726706764914, "Regression_eta": 0.023098483952608974, "Regression_sin_phi": 0.01845042272047563, "Regression_cos_phi": 0.01897773959419944, "Regression_energy": 0.2210256056352095, "Total": 7.5767284046519885, "group/cld/Classification": 0.02369180385102617, "group/cld/Classification_binary": 3.2927703857421875, "group/cld/Regression_cos_phi": 0.010195688998445552, "group/cld/Regression_energy": 0.21129009571481258, "group/cld/Regression_eta": 0.01833410973244525, "group/cld/Regression_pt": 0.20499180732889377, "group/cld/Regression_sin_phi": 0.009046065046432171, "group/cld/Total": 7.353452641913232, "group/cld/Total_unweighted": 3.7703199564142427, "group/clic/Classification": 0.027257990329823595, "group/clic/Classification_binary": 3.0790863037109375, "group/clic/Regression_cos_phi": 0.010964055010612974, "group/clic/Regression_energy": 0.17615655127992022, "group/clic/Regression_eta": 0.015102987593792855, "group/clic/Regression_pt": 0.17099656449987533, "group/clic/Regression_sin_phi": 0.010694893116646625, "group/clic/Total": 6.633870875581782, "group/clic/Total_unweighted": 3.490259345541609, "group/colliderml/Classification": 0.025630369782447815, "group/colliderml/Classification_binary": 2.7437357902526855, "group/colliderml/Regression_cos_phi": 0.06831520050764084, "group/colliderml/Regression_energy": 0.3814266622066498, "group/colliderml/Regression_eta": 0.060580573976039886, "group/colliderml/Regression_pt": 0.3458652198314667, "group/colliderml/Regression_sin_phi": 0.06885762512683868, "group/colliderml/Total": 11.00226879119873, "group/colliderml/Total_unweighted": 3.6944114416837692, "Total_unweighted": 3.639615967056968, "metrics/particle/class_1/efficiency": 0.9355541467666626, "metrics/particle/class_1/pid_accuracy": 0.9773080348968506, "metrics/particle/class_2/efficiency": 0.19391122460365295, "metrics/particle/class_2/pid_accuracy": 0.48667412996292114, "metrics/particle/class_3/efficiency": 0.6072900295257568, "metrics/particle/class_3/pid_accuracy": 0.9695178270339966, "metrics/particle/class_4/efficiency": 0.8667199015617371, "metrics/particle/class_4/pid_accuracy": 0.16850829124450684, "metrics/particle/class_5/efficiency": 0.9653679728507996, "metrics/particle/class_5/pid_accuracy": 0.4708520174026489, "metrics/particle/count/bias_mean": -3.5411429405212402, "metrics/particle/count/mae": 6.193714141845703, "metrics/particle/count/prediction_mean": 88.61314392089844, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.06612349301576614, "metrics/particle/event/energy_response_mean": 0.97308748960495, "metrics/particle/event/met_abs_error": 8.232911109924316, "metrics/particle/event/scalar_pt_relative_abs_error": 0.053395602852106094, "metrics/particle/event/scalar_pt_response_mean": 0.9855884313583374, "metrics/particle/event/vector_pt_closure": 0.056844159960746765, "metrics/particle/matched/delta_eta_abs_mean": 0.020078372210264206, "metrics/particle/matched/delta_phi_abs_mean": 0.017527084797620773, "metrics/particle/matched/delta_r_mean": 0.02976417727768421, "metrics/particle/matched/energy_relative_abs_mean": 0.09211724251508713, "metrics/particle/matched/pid_accuracy": 0.9433226585388184, "metrics/particle/matched/pt_relative_abs_mean": 0.08813007175922394, "metrics/particle/matching/duplicate_fraction": 0.0396716371178627, "metrics/particle/matching/efficiency": 0.6835617423057556, "metrics/particle/matching/f1": 0.6969523429870605, "metrics/particle/matching/purity": 0.7108780741691589, "metrics/particle/matching/target_coverage_dr0p05": 0.5714392066001892, "metrics/particle/matching/target_coverage_dr0p10": 0.7139703631401062, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6194642782211304, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7732188105583191}, "step_train_time": 7.390488386154175, "step_valid_time": 12.43390773399733, "step_total_time": 10024.4769551754, "elapsed_training_loop_seconds": 10024.4769551754, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9942049898891707, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12931237558119513, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1376, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1364, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9259757738896366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9347826086956522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9303583502366464, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06521739130434782, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.917900403768506, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9266304347826086, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9222447599729547, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9259757738896366, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9894469287295301, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13233859216068333, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1107, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1010, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1001, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9033989266547406, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.912375790424571, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9078651685393259, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08762420957542909, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8953488372093024, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9042457091237579, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8997752808988764, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9033989266547406, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994335794383763, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12555754069609937, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 623, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 527, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 524, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.839171974522293, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8459069020866774, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8425259792166268, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15409309791332262, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8343949044585988, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.841091492776886, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8377298161470823, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.839171974522293, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9946903141659584, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09880982375302094, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1406, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1402, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9423592493297587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9474393530997305, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9448924731182796, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05256064690026954, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9396782841823056, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.944743935309973, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9422043010752689, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9423592493297587, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9878687440224008, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10937085975450955, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1100, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1002, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 999, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9018901890189019, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9109090909090909, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9063772048846676, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0890909090909091, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8991899189918992, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9081818181818182, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.903663500678426, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9018901890189019, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9856284456585759, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10434105899847734, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 637, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 551, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 547, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8663522012578616, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8649921507064364, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8656716417910447, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13500784929356358, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.860062893081761, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.858712715855573, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8593872741555381, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8663522012578616, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9762181550613653, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20283342381085767, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3481, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2601, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2541, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7332957428813082, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7471990807239299, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7401821286283438, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.25280091927607007, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7163800394699746, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7299626544096524, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7231075697211156, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7332957428813082, "step_test_time": 33.973916206974536, "step_plot_time": 26.652925200061873, "step_validation_cycle_time": 73.57152290293016} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_90000.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_90000.json new file mode 100644 index 0000000000000000000000000000000000000000..db609bd7499e8e4658e71a618d3d0468d12fbea4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/history/step_90000.json @@ -0,0 +1 @@ +{"epoch": 0.22841769983376267, "train": {"Classification_binary": 2.5929858684539795, "Classification": 0.022432440891861916, "Regression_pt": 0.37471649050712585, "Regression_eta": 0.06098563224077225, "Regression_sin_phi": 0.07304806262254715, "Regression_cos_phi": 0.06962306052446365, "Regression_energy": 0.40843239426612854, "Total": 11.33830451965332, "Total_unweighted": 3.602223949506879}, "train_group": {"cld": {"Classification_binary": 3.2225295795692697, "Classification": 0.023243621363676525, "Regression_pt": 0.19431718130100956, "Regression_eta": 0.019067100484159077, "Regression_sin_phi": 0.009760008560627833, "Regression_cos_phi": 0.009914139104593132, "Regression_energy": 0.20176542677857853, "Total": 7.119435890300854, "Total_unweighted": 3.6805970571619144}, "clic": {"Classification_binary": 2.9642897210679613, "Classification": 0.02562952258654901, "Regression_pt": 0.1637908165698295, "Regression_eta": 0.012938311850713627, "Regression_sin_phi": 0.011209894552010225, "Regression_cos_phi": 0.011296814760646305, "Regression_energy": 0.1656290777810701, "Total": 6.354846062244953, "Total_unweighted": 3.35478415916878}, "colliderml": {"Classification_binary": 2.679456363894982, "Classification": 0.02541734749440126, "Regression_pt": 0.3376671037452664, "Regression_eta": 0.06208499321368283, "Regression_sin_phi": 0.06992127110590478, "Regression_cos_phi": 0.069762241797979, "Regression_energy": 0.3804107891942213, "Total": 10.886672231251609, "Total_unweighted": 3.6247201104464377}}, "valid": {"Classification_binary": 3.033210615678267, "Classification": 0.024640141833912242, "Regression_pt": 0.20400189486416903, "Regression_eta": 0.021568437056107954, "Regression_sin_phi": 0.018313030763105913, "Regression_cos_phi": 0.018406349962407892, "Regression_energy": 0.21362634138627487, "Total": 7.342390025745739, "group/cld/Classification": 0.02290450004821128, "group/cld/Classification_binary": 3.1959426555227726, "group/cld/Regression_cos_phi": 0.009710654299309913, "group/cld/Regression_energy": 0.20383617725778133, "group/cld/Regression_eta": 0.016646667997887794, "group/cld/Regression_pt": 0.1976623129337392, "group/cld/Regression_sin_phi": 0.008925103126688207, "group/cld/Total": 7.107883534532912, "group/cld/Total_unweighted": 3.65562807118639, "group/clic/Classification": 0.026308757193545078, "group/clic/Classification_binary": 2.994977910467919, "group/clic/Regression_cos_phi": 0.010386827144216983, "group/clic/Regression_energy": 0.17002481095334318, "group/clic/Regression_eta": 0.0131322665417448, "group/clic/Regression_pt": 0.1653002475170379, "group/clic/Regression_sin_phi": 0.01065538411444806, "group/clic/Total": 6.421805523811503, "group/clic/Total_unweighted": 3.390786203932255, "group/colliderml/Classification": 0.024837050586938858, "group/colliderml/Classification_binary": 2.667494535446167, "group/colliderml/Regression_cos_phi": 0.0675073117017746, "group/colliderml/Regression_energy": 0.37046462297439575, "group/colliderml/Regression_eta": 0.060807399451732635, "group/colliderml/Regression_pt": 0.3363102674484253, "group/colliderml/Regression_sin_phi": 0.06838440150022507, "group/colliderml/Total": 10.735474586486816, "group/colliderml/Total_unweighted": 3.595805589109659, "Total_unweighted": 3.533766811544245, "metrics/particle/class_1/efficiency": 0.9394323229789734, "metrics/particle/class_1/pid_accuracy": 0.9803292751312256, "metrics/particle/class_2/efficiency": 0.2055068165063858, "metrics/particle/class_2/pid_accuracy": 0.5230346322059631, "metrics/particle/class_3/efficiency": 0.6117886900901794, "metrics/particle/class_3/pid_accuracy": 0.972687840461731, "metrics/particle/class_4/efficiency": 0.8922585844993591, "metrics/particle/class_4/pid_accuracy": 0.1824686974287033, "metrics/particle/class_5/efficiency": 0.9653679728507996, "metrics/particle/class_5/pid_accuracy": 0.4857997000217438, "metrics/particle/count/bias_mean": -2.7131428718566895, "metrics/particle/count/mae": 6.163428783416748, "metrics/particle/count/prediction_mean": 89.4411392211914, "metrics/particle/count/target_mean": 92.15428924560547, "metrics/particle/event/energy_relative_abs_error": 0.0652746632695198, "metrics/particle/event/energy_response_mean": 0.973101019859314, "metrics/particle/event/met_abs_error": 8.190224647521973, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05323738604784012, "metrics/particle/event/scalar_pt_response_mean": 0.9827417731285095, "metrics/particle/event/vector_pt_closure": 0.05702311173081398, "metrics/particle/matched/delta_eta_abs_mean": 0.01750054955482483, "metrics/particle/matched/delta_phi_abs_mean": 0.016096604987978935, "metrics/particle/matched/delta_r_mean": 0.026517389342188835, "metrics/particle/matched/energy_relative_abs_mean": 0.09441367536783218, "metrics/particle/matched/pid_accuracy": 0.9468364715576172, "metrics/particle/matched/pt_relative_abs_mean": 0.08970946073532104, "metrics/particle/matching/duplicate_fraction": 0.038844380527734756, "metrics/particle/matching/efficiency": 0.6889688372612, "metrics/particle/matching/f1": 0.6992624402046204, "metrics/particle/matching/purity": 0.7098682522773743, "metrics/particle/matching/target_coverage_dr0p05": 0.5996155738830566, "metrics/particle/matching/target_coverage_dr0p10": 0.7252247929573059, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6172443628311157, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7742295265197754}, "step_train_time": 7.372549057006836, "step_valid_time": 12.37895501696039, "step_total_time": 11285.654173374176, "elapsed_training_loop_seconds": 11285.654173374176, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9818187031854066, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12636960836077105, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1379, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1372, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9279946164199192, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9368206521739131, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9323867478025692, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06317934782608696, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9232839838492598, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9320652173913043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9276538201487492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9279946164199192, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9861068691837859, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1257790389159148, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1113, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1030, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1019, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9212880143112702, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9254267744833783, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.923352756611385, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07457322551662174, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9114490161001789, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9155435759209344, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9134917077543703, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9212880143112702, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9976433880827338, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12335083941069458, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 620, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 547, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 543, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8710191082802548, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.882258064516129, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8766025641025641, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11774193548387096, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8646496815286624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8758064516129033, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8701923076923077, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8710191082802548, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9811858098198109, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08834141637631687, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1395, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9376675603217158, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9439946018893387, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9408204438466712, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05600539811066127, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9349865951742627, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9412955465587044, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9381304640215198, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9376675603217158, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9807773863158836, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10094413063749674, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1101, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1021, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1017, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.918991899189919, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.927338782924614, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9231464737793852, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07266121707538602, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9153915391539154, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9237057220708447, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9195298372513563, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.918991899189919, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9851456559188433, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1029090125196106, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 549, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 548, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8632075471698113, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8645669291338582, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8638867033831629, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13543307086614173, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8616352201257862, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.862992125984252, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8623131392604249, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8632075471698113, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9746044459932017, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18477664332330135, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3547, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3550, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2645, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2597, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7457005920496194, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7450704225352113, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7453853741017331, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2549295774647887, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7321680293205526, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7315492957746479, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.731858531773989, "step/colliderml_ttbar_nopu_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7457005920496194, "step_test_time": 34.03728086000774, "step_plot_time": 26.612832951010205, "step_validation_cycle_time": 73.53431110701058} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..4545edc3063d371d41742e8a5dfbfb3767097116 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": {"cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/"}, "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 1000, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"cld": {"batch_size": 1, "weight": 1.0, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}, "clic": {"batch_size": 1, "weight": 1.0, "samples": {"clic_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}, "colliderml": {"batch_size": 1, "weight": 1.0, "samples": {"colliderml_ttbar_nopu_pf": {"version": "1.5.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"cld": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}, "clic": {"batch_size": 1, "weight": null, "samples": {"clic_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}, "colliderml": {"batch_size": 1, "weight": null, "samples": {"colliderml_ttbar_nopu_pf": {"version": "1.5.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "colliderml_ttbar_nopu_pf": {"version": "1.5.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf", "cld_edm_ww_fullhad_pf", "cld_edm_qq_pf", "clic_edm_ttbar_pf", "clic_edm_ww_fullhad_pf", "clic_edm_qq_pf", "colliderml_ttbar_nopu_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..993a5d1056c7b706fc974b9eedf56c8006b34474 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19ce5a5f3c9bc8303acd1e858ed12eedeff04f6832858bb2b08f03b72d3af86b +size 7366 diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..801fe86b4414aa9ef7bbf2108a89a8e6a00daf96 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..d23c3102334a8ddbc197249d40ac68240f2110cb Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ccabd52671555c131df0263f8d4de1bcb16cbf7a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..913f4da549493390bcf7c9d518d2cc894949e3b2 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..72d83e6e77bfc9a5c3c8c5cd656eadc3bda0851d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..dc509a1acafe1ff483f771219fa1d8dae96f462a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..55d77d0a1ac1017d41513838116c0b2cbf5b10ac Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..79b5c54b52c871e704e912106f423d06e295a107 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..14b2bcb0090ce66d064c940b0f5a9d767f6508df Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..ee6234955ec168b071d582cd95a0d7e695c1b27a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b5673f90a9065152fa64ca356d645515ef70212c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..63a7cbec68c9eb3cce3b183ac6684c5bfaf50e6a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_qq_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dc9e23535898363268f04f6394c1605b4dcac9fb Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..4a31a938fa55eac20939a9e8f5b6b054a5a38575 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7afc468c9bd691d285676ef9dabba33582c73bcf Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..00f79633ccafd5287eb423ea1198caec683bf7ef Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c9fb9ee3b00a029d14c584d657687f762be3bc32 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..a3ae41fba09a980e54b26b04131ce880a32c2db4 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c3bcd0d3b02a77c4f00b7ce7028dc13b7ad55a66 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..a66065b5031a2ce49d558532541616a62e49b240 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2932752d40aa76105dd176599f149d97f91c551e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..1c14a37027936962de3521de4902976dc007f17e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a83dc7d56cd95b22e4fd1c2ac0fc58cc67255453 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..3bac26a0b6c7b73dcc486455b17438c28cb7961d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_joint/joint_seed12345_20261114_063930_566389/plots_step_500000/cld_edm_ttbar_pf/num_elements.png differ