diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..9172018f225ccc15bbe3622f652a3fa3910459ae --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f516c5ed88f2f000355e32801d8533c56bdda6768fa937442fe2040af9484d14 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-50000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..f1ca929e14df52cbd57331adfc1a6a11ccf9fc9d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3916e6116e80e23e4c298a7a018c7a14bb6f98fa4138dd7f4ccaff74c7d09e62 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..0281216667bda0bf9614cb9f22962bebd431772a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 2.470405340194702, "Classification": 0.01498999260365963, "Regression_pt": 0.2600608170032501, "Regression_eta": 0.006616869010031223, "Regression_sin_phi": 0.004621132742613554, "Regression_cos_phi": 0.004002876579761505, "Regression_energy": 0.2713663876056671, "Total": 6.39364767074585, "Total_unweighted": 3.0320634157396853}, "train_group": {}, "valid": {"Classification_binary": 2.4950942993164062, "Classification": 0.012887252494692802, "Regression_pt": 0.20939947664737701, "Regression_eta": 0.014357101172208786, "Regression_sin_phi": 0.009900463744997978, "Regression_cos_phi": 0.008746917359530926, "Regression_energy": 0.20825736224651337, "Total": 5.769011974334717, "Total_unweighted": 2.958642872981727, "metrics/particle/class_1/efficiency": 0.863043487071991, "metrics/particle/class_1/pid_accuracy": 0.9722921848297119, "metrics/particle/class_2/efficiency": 0.2380952388048172, "metrics/particle/class_2/pid_accuracy": 0.6000000238418579, "metrics/particle/class_3/efficiency": 0.512499988079071, "metrics/particle/class_3/pid_accuracy": 0.9756097793579102, "metrics/particle/class_4/efficiency": 0.8383838534355164, "metrics/particle/class_4/pid_accuracy": 0.9397590160369873, "metrics/particle/class_5/efficiency": 0.9292929172515869, "metrics/particle/class_5/pid_accuracy": 0.989130437374115, "metrics/particle/count/bias_mean": -0.1120000034570694, "metrics/particle/count/mae": 0.6320000290870667, "metrics/particle/count/prediction_mean": 4.52400016784668, "metrics/particle/count/target_mean": 4.636000156402588, "metrics/particle/event/energy_relative_abs_error": 0.08760278671979904, "metrics/particle/event/energy_response_mean": 0.9866527915000916, "metrics/particle/event/met_abs_error": 2.8842148780822754, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08855906873941422, "metrics/particle/event/scalar_pt_response_mean": 0.9902690649032593, "metrics/particle/event/vector_pt_closure": 0.1092691421508789, "metrics/particle/matched/delta_eta_abs_mean": 0.0280148983001709, "metrics/particle/matched/delta_phi_abs_mean": 0.024209514260292053, "metrics/particle/matched/delta_r_mean": 0.0412246435880661, "metrics/particle/matched/energy_relative_abs_mean": 0.10055454820394516, "metrics/particle/matched/pid_accuracy": 0.969623327255249, "metrics/particle/matched/pt_relative_abs_mean": 0.10068883746862411, "metrics/particle/matching/duplicate_fraction": 0.029177719727158546, "metrics/particle/matching/efficiency": 0.7100949287414551, "metrics/particle/matching/f1": 0.7187772989273071, "metrics/particle/matching/purity": 0.7276746034622192, "metrics/particle/matching/target_coverage_dr0p05": 0.48662641644477844, "metrics/particle/matching/target_coverage_dr0p10": 0.7506471276283264, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6617773771286011, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8127696514129639}, "step_train_time": 0.6028058528900146, "step_valid_time": 2.8203783889766783, "step_total_time": 1113.3894712924957, "elapsed_training_loop_seconds": 1113.3894712924957, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0074353931865176, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09821468632867425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 395, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9244851258581236, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9287356321839081, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.926605504587156, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07126436781609195, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9038901601830663, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9080459770114943, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9059633027522935, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9244851258581236, "step_test_time": 5.576461191056296, "step_plot_time": 4.444414507015608, "step_validation_cycle_time": 13.37055923906155} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..f48fe51252a7a3c9c29590c45430a3ef4e991439 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 1.9142053127288818, "Classification": 0.002808774821460247, "Regression_pt": 0.18370336294174194, "Regression_eta": 0.006747966166585684, "Regression_sin_phi": 0.009045658633112907, "Regression_cos_phi": 0.004843755625188351, "Regression_energy": 0.18506161868572235, "Total": 4.634146213531494, "Total_unweighted": 2.3064164496026933}, "train_group": {}, "valid": {"Classification_binary": 2.359476089477539, "Classification": 0.01065006759017706, "Regression_pt": 0.20814518630504608, "Regression_eta": 0.010532179847359657, "Regression_sin_phi": 0.008639832958579063, "Regression_cos_phi": 0.006208475679159164, "Regression_energy": 0.21246372163295746, "Total": 5.560715198516846, "Total_unweighted": 2.8161155534908175, "metrics/particle/class_1/efficiency": 0.897826075553894, "metrics/particle/class_1/pid_accuracy": 0.9782082438468933, "metrics/particle/class_2/efficiency": 0.3333333432674408, "metrics/particle/class_2/pid_accuracy": 0.5714285969734192, "metrics/particle/class_3/efficiency": 0.5770833492279053, "metrics/particle/class_3/pid_accuracy": 0.9747292399406433, "metrics/particle/class_4/efficiency": 0.8383838534355164, "metrics/particle/class_4/pid_accuracy": 0.9156626462936401, "metrics/particle/class_5/efficiency": 0.9797979593276978, "metrics/particle/class_5/pid_accuracy": 0.9896907210350037, "metrics/particle/count/bias_mean": 0.02800000086426735, "metrics/particle/count/mae": 0.5879999995231628, "metrics/particle/count/prediction_mean": 4.664000034332275, "metrics/particle/count/target_mean": 4.636000156402588, "metrics/particle/event/energy_relative_abs_error": 0.08774281293153763, "metrics/particle/event/energy_response_mean": 0.9923677444458008, "metrics/particle/event/met_abs_error": 2.612107753753662, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08549340069293976, "metrics/particle/event/scalar_pt_response_mean": 0.997454047203064, "metrics/particle/event/vector_pt_closure": 0.09128881245851517, "metrics/particle/matched/delta_eta_abs_mean": 0.02574164606630802, "metrics/particle/matched/delta_phi_abs_mean": 0.021925149485468864, "metrics/particle/matched/delta_r_mean": 0.03762231022119522, "metrics/particle/matched/energy_relative_abs_mean": 0.09812948107719421, "metrics/particle/matched/pid_accuracy": 0.9692132472991943, "metrics/particle/matched/pt_relative_abs_mean": 0.0980519950389862, "metrics/particle/matching/duplicate_fraction": 0.03344768285751343, "metrics/particle/matching/efficiency": 0.756686806678772, "metrics/particle/matching/f1": 0.754408597946167, "metrics/particle/matching/purity": 0.7521440982818604, "metrics/particle/matching/target_coverage_dr0p05": 0.5806729793548584, "metrics/particle/matching/target_coverage_dr0p10": 0.8058671355247498, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6833477020263672, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8360655903816223}, "step_train_time": 0.6147933006286621, "step_valid_time": 2.9689863089006394, "step_total_time": 2240.7805354595184, "elapsed_training_loop_seconds": 2240.7805354595184, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9828551354348452, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08326922599868103, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 408, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 403, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9336384439359268, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9400921658986175, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9368541905855339, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.059907834101382486, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9221967963386728, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9285714285714286, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9253731343283582, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9336384439359268, "step_test_time": 4.8475582679966465, "step_plot_time": 3.2590056729968637, "step_validation_cycle_time": 11.620401413994841} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..23e6853ef3251f88571a1b849b6c3021bf175f1c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 2.329976797103882, "Classification": 0.006666332017630339, "Regression_pt": 0.1582590639591217, "Regression_eta": 0.006296627689152956, "Regression_sin_phi": 0.0027599530294537544, "Regression_cos_phi": 0.003585765603929758, "Regression_energy": 0.11865943670272827, "Total": 4.44908332824707, "Total_unweighted": 2.6262039761058986}, "train_group": {}, "valid": {"Classification_binary": 2.6239399909973145, "Classification": 0.012073090299963951, "Regression_pt": 0.22017677128314972, "Regression_eta": 0.009221618995070457, "Regression_sin_phi": 0.008007444441318512, "Regression_cos_phi": 0.006201080046594143, "Regression_energy": 0.2277200073003769, "Total": 6.011784076690674, "Total_unweighted": 3.107340003363788, "metrics/particle/class_1/efficiency": 0.8999999761581421, "metrics/particle/class_1/pid_accuracy": 0.9758453965187073, "metrics/particle/class_2/efficiency": 0.1428571492433548, "metrics/particle/class_2/pid_accuracy": 0.6666666865348816, "metrics/particle/class_3/efficiency": 0.581250011920929, "metrics/particle/class_3/pid_accuracy": 0.9605734944343567, "metrics/particle/class_4/efficiency": 0.8888888955116272, "metrics/particle/class_4/pid_accuracy": 0.8863636255264282, "metrics/particle/class_5/efficiency": 0.9898989796638489, "metrics/particle/class_5/pid_accuracy": 0.9693877696990967, "metrics/particle/count/bias_mean": 0.052000001072883606, "metrics/particle/count/mae": 0.6200000047683716, "metrics/particle/count/prediction_mean": 4.688000202178955, "metrics/particle/count/target_mean": 4.636000156402588, "metrics/particle/event/energy_relative_abs_error": 0.10221365094184875, "metrics/particle/event/energy_response_mean": 1.0306388139724731, "metrics/particle/event/met_abs_error": 3.0977890491485596, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09756720811128616, "metrics/particle/event/scalar_pt_response_mean": 1.0288279056549072, "metrics/particle/event/vector_pt_closure": 0.10402042418718338, "metrics/particle/matched/delta_eta_abs_mean": 0.024498332291841507, "metrics/particle/matched/delta_phi_abs_mean": 0.019895203411579132, "metrics/particle/matched/delta_r_mean": 0.03521939739584923, "metrics/particle/matched/energy_relative_abs_mean": 0.09077376127243042, "metrics/particle/matched/pid_accuracy": 0.9603174328804016, "metrics/particle/matched/pt_relative_abs_mean": 0.08867111057043076, "metrics/particle/matching/duplicate_fraction": 0.03498293459415436, "metrics/particle/matching/efficiency": 0.7610008716583252, "metrics/particle/matching/f1": 0.7567567825317383, "metrics/particle/matching/purity": 0.7525597214698792, "metrics/particle/matching/target_coverage_dr0p05": 0.5953407883644104, "metrics/particle/matching/target_coverage_dr0p10": 0.795513391494751, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7066436409950256, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8386540412902832}, "step_train_time": 0.5930485725402832, "step_valid_time": 2.9109891100088134, "step_total_time": 3369.930011987686, "elapsed_training_loop_seconds": 3369.930011987686, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9987664713387776, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09312806262645335, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 406, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 394, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9290617848970252, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9311926605504587, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9301260022909508, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06880733944954129, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9016018306636155, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9036697247706422, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9026345933562429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9290617848970252, "step_test_time": 4.91679990501143, "step_plot_time": 3.2464390649693087, "step_validation_cycle_time": 11.536785959964618} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..fbf18672b4cee4bc304d861b081b8df2e973e7c7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 0.500719428062439, "Classification": 0.002525398973375559, "Regression_pt": 0.06104760989546776, "Regression_eta": 0.007465233560651541, "Regression_sin_phi": 0.00578747782856226, "Regression_cos_phi": 0.002875670325011015, "Regression_energy": 0.06672719866037369, "Total": 1.5229731798171997, "Total_unweighted": 0.6471480173058808}, "train_group": {}, "valid": {"Classification_binary": 4.046369552612305, "Classification": 0.015168276615440845, "Regression_pt": 0.2710207402706146, "Regression_eta": 0.009634613990783691, "Regression_sin_phi": 0.007051053922623396, "Regression_cos_phi": 0.006012238096445799, "Regression_energy": 0.2764267325401306, "Total": 8.147441864013672, "Total_unweighted": 4.631683208048344, "metrics/particle/class_1/efficiency": 0.9021739363670349, "metrics/particle/class_1/pid_accuracy": 0.966265082359314, "metrics/particle/class_2/efficiency": 0.2857142984867096, "metrics/particle/class_2/pid_accuracy": 0.6666666865348816, "metrics/particle/class_3/efficiency": 0.5333333611488342, "metrics/particle/class_3/pid_accuracy": 0.95703125, "metrics/particle/class_4/efficiency": 0.8888888955116272, "metrics/particle/class_4/pid_accuracy": 0.9204545617103577, "metrics/particle/class_5/efficiency": 0.9898989796638489, "metrics/particle/class_5/pid_accuracy": 0.9897959232330322, "metrics/particle/count/bias_mean": 0.024000000208616257, "metrics/particle/count/mae": 0.671999990940094, "metrics/particle/count/prediction_mean": 4.659999847412109, "metrics/particle/count/target_mean": 4.636000156402588, "metrics/particle/event/energy_relative_abs_error": 0.10295739769935608, "metrics/particle/event/energy_response_mean": 0.9809581637382507, "metrics/particle/event/met_abs_error": 3.1130566596984863, "metrics/particle/event/scalar_pt_relative_abs_error": 0.10064424574375153, "metrics/particle/event/scalar_pt_response_mean": 0.9849731922149658, "metrics/particle/event/vector_pt_closure": 0.10102332383394241, "metrics/particle/matched/delta_eta_abs_mean": 0.024862930178642273, "metrics/particle/matched/delta_phi_abs_mean": 0.018575577065348625, "metrics/particle/matched/delta_r_mean": 0.03451808542013168, "metrics/particle/matched/energy_relative_abs_mean": 0.0863056629896164, "metrics/particle/matched/pid_accuracy": 0.9594438076019287, "metrics/particle/matched/pt_relative_abs_mean": 0.08467335253953934, "metrics/particle/matching/duplicate_fraction": 0.03261802718043327, "metrics/particle/matching/efficiency": 0.7446074485778809, "metrics/particle/matching/f1": 0.7426850199699402, "metrics/particle/matching/purity": 0.740772545337677, "metrics/particle/matching/target_coverage_dr0p05": 0.5789473652839661, "metrics/particle/matching/target_coverage_dr0p10": 0.7739430665969849, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.693701446056366, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8352027535438538}, "step_train_time": 0.5716068744659424, "step_valid_time": 2.9506208589300513, "step_total_time": 4500.595416307449, "elapsed_training_loop_seconds": 4500.595416307449, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9898951446376003, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07098580459097559, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 406, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 395, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9290617848970252, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9419953596287703, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9354838709677419, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.058004640371229696, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9038901601830663, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9164733178654292, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9101382488479263, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9290617848970252, "step_test_time": 4.93864590395242, "step_plot_time": 3.2144215418957174, "step_validation_cycle_time": 11.535477022989653} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..6795c0d7f3ebbd13478fba21f6f7d0e125080d20 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 0.669297456741333, "Classification": 0.0003745425201486796, "Regression_pt": 0.0415232814848423, "Regression_eta": 0.0017013844335451722, "Regression_sin_phi": 0.0016055343439802527, "Regression_cos_phi": 0.0015140834730118513, "Regression_energy": 0.038813576102256775, "Total": 1.2645851373672485, "Total_unweighted": 0.754829859099118}, "train_group": {}, "valid": {"Classification_binary": 6.121794700622559, "Classification": 0.023892665281891823, "Regression_pt": 0.2646413743495941, "Regression_eta": 0.009020299650728703, "Regression_sin_phi": 0.0070997984148561954, "Regression_cos_phi": 0.005211640149354935, "Regression_energy": 0.27120810747146606, "Total": 10.243091583251953, "Total_unweighted": 6.70286858594045, "metrics/particle/class_1/efficiency": 0.9108695387840271, "metrics/particle/class_1/pid_accuracy": 0.9809069037437439, "metrics/particle/class_2/efficiency": 0.2380952388048172, "metrics/particle/class_2/pid_accuracy": 0.4000000059604645, "metrics/particle/class_3/efficiency": 0.5062500238418579, "metrics/particle/class_3/pid_accuracy": 0.9794238805770874, "metrics/particle/class_4/efficiency": 0.8888888955116272, "metrics/particle/class_4/pid_accuracy": 0.8977272510528564, "metrics/particle/class_5/efficiency": 0.9898989796638489, "metrics/particle/class_5/pid_accuracy": 0.9897959232330322, "metrics/particle/count/bias_mean": 0.12399999797344208, "metrics/particle/count/mae": 0.6359999775886536, "metrics/particle/count/prediction_mean": 4.760000228881836, "metrics/particle/count/target_mean": 4.636000156402588, "metrics/particle/event/energy_relative_abs_error": 0.08591156452894211, "metrics/particle/event/energy_response_mean": 0.9860177040100098, "metrics/particle/event/met_abs_error": 2.8056294918060303, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08281180262565613, "metrics/particle/event/scalar_pt_response_mean": 0.9875185489654541, "metrics/particle/event/vector_pt_closure": 0.0968654677271843, "metrics/particle/matched/delta_eta_abs_mean": 0.02260054089128971, "metrics/particle/matched/delta_phi_abs_mean": 0.018529651686549187, "metrics/particle/matched/delta_r_mean": 0.03262568265199661, "metrics/particle/matched/energy_relative_abs_mean": 0.0773727297782898, "metrics/particle/matched/pid_accuracy": 0.969519317150116, "metrics/particle/matched/pt_relative_abs_mean": 0.07580088824033737, "metrics/particle/matching/duplicate_fraction": 0.029411764815449715, "metrics/particle/matching/efficiency": 0.7359793186187744, "metrics/particle/matching/f1": 0.7262665033340454, "metrics/particle/matching/purity": 0.7168067097663879, "metrics/particle/matching/target_coverage_dr0p05": 0.596203625202179, "metrics/particle/matching/target_coverage_dr0p10": 0.7704917788505554, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7006039619445801, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8438308835029602}, "step_train_time": 0.5911049842834473, "step_valid_time": 2.9122192590730265, "step_total_time": 5630.75192284584, "elapsed_training_loop_seconds": 5630.75192284584, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9964882214744637, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05293799171410429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 437, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 412, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 405, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9427917620137299, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9427917620137299, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9427917620137299, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.057208237986270026, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9267734553775744, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9267734553775744, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9267734553775744, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9427917620137299, "step_test_time": 4.780989658087492, "step_plot_time": 3.141829597996548, "step_validation_cycle_time": 11.239615236991085} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..7df696abbf5bd0d1d65dae771692a46d0f81ca08 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_1p5mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_z_tautau_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..eaa64b741ab2d3c8780c032f87aabc77a236116b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:173683bc55e375df5711f1dbd76c7723ed41133aff628c51c670079916191d20 +size 4404 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..df13f8c59f0818399b5fa8ad9ec9a6d3b0e957ab Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..2eeaa417c7c1265c8e44211bea58f3141ad21557 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6619c013fa5376dd223d4944f0c252f079fef1a3 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..74d6c319b5ffeed00f3cac50e4497c5bcd462e6f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ab77956256e48085172daf9f7ea11ec27f0f5d64 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..54c3508b2bea3ef54911079e2bc3027ac70d464c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2e796b51295e7ee56ed81998cec059b296dddca6 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..6e929dd5731c39ddb87cf086cb4be6e9c0f0807b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..535af9c90be92cdf76f8dc18e4444460ca518e2d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..5895d2e0d15fbeaa84350daeaeb4c82f375199b6 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ca111ad63e8d8bd4b1f9971e6677fcf47f6160e3 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..83bb79e5370ce90c7049ecbd8d75853b59617072 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/runs/train/events.out.tfevents.1794646693.workergpu159.1868450.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/runs/train/events.out.tfevents.1794646693.workergpu159.1868450.0 new file mode 100644 index 0000000000000000000000000000000000000000..09f0862c131d56ca1059ae4642597a3fcba4d3da --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/runs/train/events.out.tfevents.1794646693.workergpu159.1868450.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2674e8100f6fbaa4af50ba5cd30d3a5a7f63d3f13bd919989e2c6ddd7749cab +size 1843423 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/runs/valid/events.out.tfevents.1794646693.workergpu159.1868450.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/runs/valid/events.out.tfevents.1794646693.workergpu159.1868450.1 new file mode 100644 index 0000000000000000000000000000000000000000..e5e2204cdb2bb84f68771cb9606dbdc35a6397af --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/runs/valid/events.out.tfevents.1794646693.workergpu159.1868450.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dff2016ceae770bb92a2110961039ca7e02f4d32e35caa5af662eea1d90f4283 +size 2325288 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..43f9f40545aca2d508e127be0a8c9e24b522e2b6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_1p5mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_1p5mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_1p5mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_1p5mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbbc90e02721d126ccfde9a08ad7d236f796b63f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_1p5mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4d5fb556d4801ade22b2ee5df7beee14ffa03a0a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/train.log @@ -0,0 +1,999 @@ +rank0 [2026-11-14 03:58:06,124] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-14 03:58:06,124] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 03:58:08,100] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 03:58:08,100] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 03:58:08,102] INFO : Instantiating model +rank0 [2026-11-14 03:58:08,136] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 03:58:08,136] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 03:58:08,138] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 03:58:08,138] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 03:58:08,138] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 03:58:08,138] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 03:58:08,138] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,140] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 03:58:08,140] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,143] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 03:58:08,143] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,146] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 03:58:08,146] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,149] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 03:58:08,149] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,151] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 03:58:08,151] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,154] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 03:58:08,154] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 03:58:08,154] INFO : Initializing output DNNs +rank0 [2026-11-14 03:58:08,167] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,169] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,177] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-14 03:58:08,177] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 03:58:08,177] INFO : jagged_attention enabled=True +rank0 [2026-11-14 03:58:08,177] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 03:58:08,177] INFO : task_queries enabled=True +rank0 [2026-11-14 03:58:08,177] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 03:58:08,177] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 03:58:08,177] INFO : classification_query parameters: 256 +rank0 [2026-11-14 03:58:08,177] INFO : regression_query parameters: 256 +rank0 [2026-11-14 03:58:08,177] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 03:58:08,177] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 03:58:08,177] INFO : output_mode=elementwise +rank0 [2026-11-14 03:58:08,177] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 03:58:08,177] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 03:58:08,177] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 03:58:08,177] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 03:58:08,177] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 03:58:08,178] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 03:58:08,178] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 03:58:08,178] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 03:58:08,179] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 03:58:08,179] INFO : MLPF __init__ done +rank0 [2026-11-14 03:58:08,179] INFO : Instantiated model +rank0 [2026-11-14 03:58:08,180] INFO : Moving model to device rank=0 +rank0 [2026-11-14 03:58:08,194] INFO : Moved model to device rank=0 +rank0 [2026-11-14 03:58:08,195] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 03:58:08,309] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 03:58:08,309] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 03:58:08,311] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 03:58:08,311] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 03:58:08,311] INFO : Total parameters: 3910930 +rank0 [2026-11-14 03:58:08,313] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 03:58:08,319] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077 +rank0 [2026-11-14 03:58:08,319] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077 +rank0 [2026-11-14 03:58:08,319] INFO : Getting dataloaders +rank0 [2026-11-14 03:58:08,319] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:58:12,974] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:13,025] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:13,081] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:13,190] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,243] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,314] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,364] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,418] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,465] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:13,517] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,517] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 03:58:13,518] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 03:58:13,518] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 03:58:13,518] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:58:13,529] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,538] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,554] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,571] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,581] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,599] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,609] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,621] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,629] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,640] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,640] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 03:58:13,640] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 03:58:13,713] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:58:13,826] INFO : Got dataloaders +rank0 [2026-11-14 03:58:13,826] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 03:58:13,826] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 03:58:13,827] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 03:58:13,834] INFO : Creating train_iterator +rank0 [2026-11-14 03:58:13,834] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 03:58:13,861] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:58:14,005] INFO : Created train_iterator +rank0 [2026-11-14 03:58:27,169] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.600406875610352, 'Classification': 0.5960752093791961, 'Regression_pt': 0.9448277205228806, 'Regression_eta': 1.273502309024334, 'Regression_sin_phi': 0.8084679882228375, 'Regression_cos_phi': 0.840230675637722, 'Regression_energy': 1.356031168103218}, weights={'Classification_binary': 1.0, 'Classification': 12.750751495361328, 'Regression_pt': 8.044224739074707, 'Regression_eta': 5.968113899230957, 'Regression_sin_phi': 9.400999069213867, 'Regression_cos_phi': 9.045619010925293, 'Regression_energy': 5.604890823364258} +rank0 [2026-11-14 03:59:19,013] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 03:59:19,014] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:00:08,136] INFO : Step 1000/500000 rank0 | Train Loss: 16.5767 | Train Loss Unweighted: 6.0048 | Train Loss Components: Classification: 0.0973 | Classification_binary: 4.3464 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6390 | Regression_eta: 0.0493 | Regression_pt: 0.7932 | Regression_sin_phi: 0.0388 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 1.2403 | Classification_binary: 4.3464 | Regression_cos_phi: 0.3683 | Regression_energy: 3.5816 | Regression_eta: 0.2941 | Regression_pt: 6.3808 | Regression_sin_phi: 0.3651 | LR: 2.01e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5777s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0355 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1942 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2566.9 | uss_MB: 1398.6 | child_rss_MB: 22726.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:00:21,818] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:00:21,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:23,884] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:01:23,884] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:58,936] INFO : Step 2000/500000 rank0 | Train Loss: 9.5641 | Train Loss Unweighted: 4.1708 | Train Loss Components: Classification: 0.0381 | Classification_binary: 3.2961 | Regression_cos_phi: 0.0152 | Regression_energy: 0.3862 | Regression_eta: 0.0223 | Regression_pt: 0.3942 | Regression_sin_phi: 0.0186 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.4863 | Classification_binary: 3.2961 | Regression_cos_phi: 0.1378 | Regression_energy: 2.1648 | Regression_eta: 0.1332 | Regression_pt: 3.1708 | Regression_sin_phi: 0.1752 | LR: 2.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5858s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0397 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.2122 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2567.6 | uss_MB: 1399.4 | child_rss_MB: 22629.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:01:58,939] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:02:26,071] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:02:26,072] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:28,005] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:03:28,005] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:49,207] INFO : Step 3000/500000 rank0 | Train Loss: 7.4587 | Train Loss Unweighted: 3.9207 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.3561 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2471 | Regression_eta: 0.0117 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.4327 | Classification_binary: 3.3561 | Regression_cos_phi: 0.0866 | Regression_energy: 1.3850 | Regression_eta: 0.0699 | Regression_pt: 1.9991 | Regression_sin_phi: 0.1293 | LR: 2.05e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5567s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1906 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.5 | uss_MB: 1402.7 | child_rss_MB: 22629.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:04:30,345] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:04:30,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:32,209] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:05:32,209] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:40,003] INFO : Step 4000/500000 rank0 | Train Loss: 7.2400 | Train Loss Unweighted: 3.2831 | Train Loss Components: Classification: 0.0523 | Classification_binary: 2.6559 | Regression_cos_phi: 0.0255 | Regression_energy: 0.2617 | Regression_eta: 0.0547 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.6665 | Classification_binary: 2.6559 | Regression_cos_phi: 0.2305 | Regression_energy: 1.4670 | Regression_eta: 0.3267 | Regression_pt: 1.7591 | Regression_sin_phi: 0.1344 | LR: 2.08e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.5717s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0396 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1974 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2570.5 | uss_MB: 1402.7 | child_rss_MB: 22628.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:05:40,005] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:06:35,060] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:06:35,060] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:30,921] INFO : Step 5000/500000 rank0 | Train Loss: 2.9962 | Train Loss Unweighted: 2.0725 | Train Loss Components: Classification: 0.0217 | Classification_binary: 1.9383 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0490 | Regression_eta: 0.0076 | Regression_pt: 0.0455 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2766 | Classification_binary: 1.9383 | Regression_cos_phi: 0.0620 | Regression_energy: 0.2744 | Regression_eta: 0.0453 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0338 | LR: 2.13e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5818s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0400 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.2028 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2568.8 | uss_MB: 1401.0 | child_rss_MB: 22626.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:07:37,404] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:07:37,404] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:39,933] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:08:39,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:21,273] INFO : Step 6000/500000 rank0 | Train Loss: 8.1058 | Train Loss Unweighted: 3.3835 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5957 | Regression_cos_phi: 0.0069 | Regression_energy: 0.3743 | Regression_eta: 0.0129 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2672 | Classification_binary: 2.5957 | Regression_cos_phi: 0.0626 | Regression_energy: 2.0979 | Regression_eta: 0.0771 | Regression_pt: 2.9566 | Regression_sin_phi: 0.0487 | LR: 2.19e-05 | DataLoad Time: 0.0025s | Model Forward Time: 0.5605s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0028 | max_backward: 0.0411 | max_optimizer_step: 0.0641 | max_tensorboard_logging: 0.2076 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.8 | uss_MB: 1401.0 | child_rss_MB: 22625.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:09:21,275] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:09:41,160] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:09:41,160] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:42,963] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:10:42,963] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:11,897] INFO : Step 7000/500000 rank0 | Train Loss: 6.8405 | Train Loss Unweighted: 3.6048 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.0728 | Regression_cos_phi: 0.0117 | Regression_energy: 0.2333 | Regression_eta: 0.0205 | Regression_pt: 0.2393 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1957 | Classification_binary: 3.0728 | Regression_cos_phi: 0.1059 | Regression_energy: 1.3077 | Regression_eta: 0.1221 | Regression_pt: 1.9252 | Regression_sin_phi: 0.1111 | LR: 2.26e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5588s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1860 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2569.8 | uss_MB: 1402.0 | child_rss_MB: 22621.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:11:45,477] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:11:45,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:12:47,693] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:12:47,693] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:02,786] INFO : Step 8000/500000 rank0 | Train Loss: 6.8290 | Train Loss Unweighted: 4.2079 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.7898 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1823 | Regression_eta: 0.0076 | Regression_pt: 0.1880 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.3238 | Classification_binary: 3.7898 | Regression_cos_phi: 0.0715 | Regression_energy: 1.0218 | Regression_eta: 0.0454 | Regression_pt: 1.5124 | Regression_sin_phi: 0.0643 | LR: 2.34e-05 | DataLoad Time: 0.0019s | Model Forward Time: 0.5877s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0355 | max_loss: 0.0028 | max_backward: 0.0392 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1980 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2569.9 | uss_MB: 1402.1 | child_rss_MB: 22618.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:13:02,788] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:13:02,867] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:13:50,206] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:13:50,207] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:52,203] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:14:52,203] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:53,534] INFO : Step 9000/500000 rank0 | Train Loss: 7.3855 | Train Loss Unweighted: 3.2611 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5667 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2656 | Regression_eta: 0.1095 | Regression_pt: 0.2830 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2350 | Classification_binary: 2.5667 | Regression_cos_phi: 0.0808 | Regression_energy: 1.4886 | Regression_eta: 0.6534 | Regression_pt: 2.2765 | Regression_sin_phi: 0.0844 | LR: 2.43e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5319s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1634 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2567.9 | uss_MB: 1400.1 | child_rss_MB: 22599.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:15:54,315] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:15:54,315] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:16:44,313] INFO : Step 10000/500000 rank0 | Train Loss: 6.3936 | Train Loss Unweighted: 3.0321 | Train Loss Components: Classification: 0.0150 | Classification_binary: 2.4704 | Regression_cos_phi: 0.0040 | Regression_energy: 0.2714 | Regression_eta: 0.0066 | Regression_pt: 0.2601 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1911 | Classification_binary: 2.4704 | Regression_cos_phi: 0.0362 | Regression_energy: 1.5210 | Regression_eta: 0.0395 | Regression_pt: 2.0920 | Regression_sin_phi: 0.0434 | LR: 2.52e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.6015s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0027 | max_backward: 0.0387 | max_optimizer_step: 0.0661 | max_tensorboard_logging: 0.2324 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.5 | uss_MB: 1400.6 | child_rss_MB: 22621.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:16:44,315] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:16:44,391] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:16:44,392] INFO : Running validation on rank0 +rank0 [2026-11-14 04:16:44,393] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:16:44,393] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:16:44,634] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:16:44,913] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:16:44,982] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:16:45,036] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:16:45,090] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:16:45,136] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:16:45,190] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:16:45,243] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:16:45,293] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:16:45,383] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:16:45,455] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:16:45,509] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:16:45,563] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:16:45,611] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:16:45,664] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:16:45,719] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:16:45,771] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:16:45,774] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:16:45,777] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:16:45,779] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:16:45,781] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:16:45,784] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:16:45,786] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:16:45,788] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:16:45,790] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:16:45,792] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:16:45,794] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:16:45,797] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:16:45,799] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:16:45,801] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:16:45,803] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:16:45,806] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:16:47,307] INFO : VALIDATION | Step=10000/500000 | Train Loss=6.3936 | Valid Loss=5.7690 | Train Loss Unweighted=3.0321 | Valid Loss Unweighted=2.9586 | Stale=0 | ETA=909.3m +rank0 [2026-11-14 04:16:47,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:16:47,902] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:16:47,916] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:16:47,973] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:16:49,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:16:49,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:16:49,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:16:49,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:16:50,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:16:50,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:16:50,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:16:50,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:16:51,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:16:51,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:16:51,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:16:51,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:16:52,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:16:52,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:16:52,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:16:52,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:16:52,881] INFO : Time taken to make predictions on device 0 is: 0.08 min +rank0 [2026-11-14 04:16:52,891] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:16:53,446] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:16:55,023] INFO : Plotted number of elements +rank0 [2026-11-14 04:16:56,364] INFO : Plotted jets +rank0 [2026-11-14 04:16:57,316] INFO : Plotted jet ratio +rank0 [2026-11-14 04:17:10,700] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:17:10,701] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:12,232] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:18:12,232] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:48,219] INFO : Step 11000/500000 rank0 | Train Loss: 6.0989 | Train Loss Unweighted: 2.7872 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.2435 | Regression_cos_phi: 0.0110 | Regression_energy: 0.2509 | Regression_eta: 0.0074 | Regression_pt: 0.2500 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2463 | Classification_binary: 2.2435 | Regression_cos_phi: 0.0999 | Regression_energy: 1.4062 | Regression_eta: 0.0440 | Regression_pt: 2.0108 | Regression_sin_phi: 0.0482 | LR: 2.63e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.6180s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.2368 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2706.0 | uss_MB: 1551.6 | child_rss_MB: 22630.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:19:14,923] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:19:14,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:17,112] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:20:17,112] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:39,615] INFO : Step 12000/500000 rank0 | Train Loss: 2.9827 | Train Loss Unweighted: 2.1655 | Train Loss Components: Classification: 0.0037 | Classification_binary: 2.0321 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0577 | Regression_eta: 0.0041 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 2.0321 | Regression_cos_phi: 0.0399 | Regression_energy: 0.3234 | Regression_eta: 0.0243 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0361 | LR: 2.75e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.5541s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1695 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2706.0 | uss_MB: 1551.6 | child_rss_MB: 22629.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:20:39,617] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:20:39,696] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:21:20,128] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:21:20,128] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:23,031] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:22:23,031] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:31,732] INFO : Step 13000/500000 rank0 | Train Loss: 5.1274 | Train Loss Unweighted: 2.9942 | Train Loss Components: Classification: 0.0308 | Classification_binary: 2.6677 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1261 | Regression_eta: 0.0097 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.3924 | Classification_binary: 2.6677 | Regression_cos_phi: 0.0711 | Regression_energy: 0.7067 | Regression_eta: 0.0579 | Regression_pt: 1.1743 | Regression_sin_phi: 0.0571 | LR: 2.88e-05 | DataLoad Time: 0.0024s | Model Forward Time: 0.5520s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0396 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1688 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2650.1 | uss_MB: 1495.8 | child_rss_MB: 22627.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:23:25,886] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:23:25,886] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:23,495] INFO : Step 14000/500000 rank0 | Train Loss: 5.4227 | Train Loss Unweighted: 3.3421 | Train Loss Components: Classification: 0.0261 | Classification_binary: 3.0139 | Regression_cos_phi: 0.0042 | Regression_energy: 0.1432 | Regression_eta: 0.0075 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.3332 | Classification_binary: 3.0139 | Regression_cos_phi: 0.0380 | Regression_energy: 0.8027 | Regression_eta: 0.0450 | Regression_pt: 1.1487 | Regression_sin_phi: 0.0412 | LR: 3.02e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5849s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0363 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.2045 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2650.6 | uss_MB: 1496.3 | child_rss_MB: 22626.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:23,497] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:24:23,580] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:24:29,170] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:24:29,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:31,600] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:25:31,601] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:15,244] INFO : Step 15000/500000 rank0 | Train Loss: 4.6705 | Train Loss Unweighted: 2.9439 | Train Loss Components: Classification: 0.0153 | Classification_binary: 2.6712 | Regression_cos_phi: 0.0039 | Regression_energy: 0.1090 | Regression_eta: 0.0057 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1949 | Classification_binary: 2.6712 | Regression_cos_phi: 0.0352 | Regression_energy: 0.6110 | Regression_eta: 0.0343 | Regression_pt: 1.0727 | Regression_sin_phi: 0.0512 | LR: 3.17e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5555s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0355 | max_loss: 0.0028 | max_backward: 0.0452 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1744 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2654.6 | uss_MB: 1500.2 | child_rss_MB: 22626.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:26:34,450] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:26:34,450] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:36,896] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:27:36,896] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:06,780] INFO : Step 16000/500000 rank0 | Train Loss: 6.2304 | Train Loss Unweighted: 3.8409 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4472 | Regression_cos_phi: 0.0043 | Regression_energy: 0.1806 | Regression_eta: 0.0114 | Regression_pt: 0.1760 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1780 | Classification_binary: 3.4472 | Regression_cos_phi: 0.0393 | Regression_energy: 1.0124 | Regression_eta: 0.0680 | Regression_pt: 1.4160 | Regression_sin_phi: 0.0696 | LR: 3.33e-05 | DataLoad Time: 0.0025s | Model Forward Time: 0.6000s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0029 | max_backward: 0.0406 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.2006 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 2654.7 | uss_MB: 1500.4 | child_rss_MB: 22622.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:28:06,782] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:06,855] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:28:39,767] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:28:39,767] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:42,004] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:29:42,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:58,125] INFO : Step 17000/500000 rank0 | Train Loss: 4.3197 | Train Loss Unweighted: 2.8574 | Train Loss Components: Classification: 0.0048 | Classification_binary: 2.6128 | Regression_cos_phi: 0.0026 | Regression_energy: 0.1149 | Regression_eta: 0.0058 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 2.6128 | Regression_cos_phi: 0.0232 | Regression_energy: 0.6438 | Regression_eta: 0.0349 | Regression_pt: 0.9030 | Regression_sin_phi: 0.0403 | LR: 3.51e-05 | DataLoad Time: 0.0018s | Model Forward Time: 0.5980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0396 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2649.6 | uss_MB: 1495.3 | child_rss_MB: 22610.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:30:45,077] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:30:45,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:47,327] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:31:47,327] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:49,508] INFO : Step 18000/500000 rank0 | Train Loss: 4.8641 | Train Loss Unweighted: 3.0079 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7212 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1058 | Regression_eta: 0.0114 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2721 | Classification_binary: 2.7212 | Regression_cos_phi: 0.0907 | Regression_energy: 0.5933 | Regression_eta: 0.0680 | Regression_pt: 1.0601 | Regression_sin_phi: 0.0588 | LR: 3.69e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5300s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0032 | max_backward: 0.0394 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1605 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2650.6 | uss_MB: 1496.2 | child_rss_MB: 22627.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:31:49,510] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:31:49,590] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:32:49,520] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:32:49,521] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:41,007] INFO : Step 19000/500000 rank0 | Train Loss: 3.3378 | Train Loss Unweighted: 1.7679 | Train Loss Components: Classification: 0.0055 | Classification_binary: 1.4958 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1365 | Regression_eta: 0.0240 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 1.4958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.7648 | Regression_eta: 0.1434 | Regression_pt: 0.7836 | Regression_sin_phi: 0.0388 | LR: 3.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 0.6656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0029 | max_backward: 0.0426 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2654.4 | uss_MB: 1500.1 | child_rss_MB: 22623.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:33:53,006] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:33:53,006] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:54,233] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:34:54,233] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:31,556] INFO : Step 20000/500000 rank0 | Train Loss: 4.6341 | Train Loss Unweighted: 2.3064 | Train Loss Components: Classification: 0.0028 | Classification_binary: 1.9142 | Regression_cos_phi: 0.0048 | Regression_energy: 0.1851 | Regression_eta: 0.0067 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 1.9142 | Regression_cos_phi: 0.0438 | Regression_energy: 1.0373 | Regression_eta: 0.0403 | Regression_pt: 1.4778 | Regression_sin_phi: 0.0850 | LR: 4.07e-05 | DataLoad Time: 0.0021s | Model Forward Time: 0.6126s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0032 | max_backward: 0.0396 | max_optimizer_step: 0.0769 | max_tensorboard_logging: 0.2090 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2654.8 | uss_MB: 1500.5 | child_rss_MB: 22657.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:31,558] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:35:31,640] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:35:31,642] INFO : Running validation on rank0 +rank0 [2026-11-14 04:35:31,642] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:35:31,642] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:35:31,768] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:35:31,876] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:35:31,985] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:35:32,057] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:35:32,126] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:35:32,194] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:35:32,262] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:35:32,331] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:35:32,437] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:35:32,545] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:35:32,652] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:35:32,721] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:35:32,788] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:35:32,855] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:35:32,926] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:35:32,996] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:35:33,113] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:35:33,118] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:35:33,121] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:35:33,124] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:35:33,126] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:35:33,128] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:35:33,130] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:35:33,132] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:35:33,134] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:35:33,136] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:35:33,137] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:35:33,139] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:35:33,141] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:35:33,144] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:35:33,147] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:35:33,148] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:35:34,698] INFO : VALIDATION | Step=20000/500000 | Train Loss=4.6341 | Valid Loss=5.5607 | Train Loss Unweighted=2.3064 | Valid Loss Unweighted=2.8161 | Stale=0 | ETA=896.3m +rank0 [2026-11-14 04:35:34,699] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:35:35,148] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:35:35,158] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:35,216] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:35:35,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:35:35,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:35:36,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:35:36,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:35:36,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:35:36,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:35:37,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:35:37,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:35:37,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:35:37,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:35:38,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:35:38,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:35:38,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:35:39,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:35:39,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:35:39,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:35:39,544] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:35:39,554] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:40,112] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:40,520] INFO : Plotted number of elements +rank0 [2026-11-14 04:35:41,828] INFO : Plotted jets +rank0 [2026-11-14 04:35:42,790] INFO : Plotted jet ratio +rank0 [2026-11-14 04:36:09,407] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:36:09,408] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:11,868] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:37:11,868] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:35,136] INFO : Step 21000/500000 rank0 | Train Loss: 4.9505 | Train Loss Unweighted: 3.5542 | Train Loss Components: Classification: 0.0069 | Classification_binary: 3.3217 | Regression_cos_phi: 0.0029 | Regression_energy: 0.1104 | Regression_eta: 0.0058 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 3.3217 | Regression_cos_phi: 0.0259 | Regression_energy: 0.6189 | Regression_eta: 0.0347 | Regression_pt: 0.8290 | Regression_sin_phi: 0.0327 | LR: 4.28e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.5840s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1796 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2682.2 | uss_MB: 1527.9 | child_rss_MB: 22629.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:38:14,786] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:38:14,786] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:17,128] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:39:17,128] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:26,672] INFO : Step 22000/500000 rank0 | Train Loss: 4.0479 | Train Loss Unweighted: 2.0364 | Train Loss Components: Classification: 0.0043 | Classification_binary: 1.6998 | Regression_cos_phi: 0.0036 | Regression_energy: 0.1563 | Regression_eta: 0.0067 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 1.6998 | Regression_cos_phi: 0.0330 | Regression_energy: 0.8762 | Regression_eta: 0.0402 | Regression_pt: 1.2606 | Regression_sin_phi: 0.0834 | LR: 4.50e-05 | DataLoad Time: 0.0021s | Model Forward Time: 0.5951s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.2146 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2682.8 | uss_MB: 1528.5 | child_rss_MB: 22628.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:39:26,674] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:39:26,756] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:40:20,194] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:40:20,194] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:18,421] INFO : Step 23000/500000 rank0 | Train Loss: 4.8737 | Train Loss Unweighted: 2.5366 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.1634 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1655 | Regression_eta: 0.0068 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.3096 | Classification_binary: 2.1634 | Regression_cos_phi: 0.0533 | Regression_energy: 0.9274 | Regression_eta: 0.0407 | Regression_pt: 1.3374 | Regression_sin_phi: 0.0420 | LR: 4.73e-05 | DataLoad Time: 0.0028s | Model Forward Time: 0.5645s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0401 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1794 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2684.8 | uss_MB: 1530.5 | child_rss_MB: 22628.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:41:23,081] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:41:23,081] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:25,950] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:42:25,950] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:10,033] INFO : Step 24000/500000 rank0 | Train Loss: 4.2370 | Train Loss Unweighted: 2.1928 | Train Loss Components: Classification: 0.0054 | Classification_binary: 1.8493 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1649 | Regression_eta: 0.0037 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 1.8493 | Regression_cos_phi: 0.0460 | Regression_energy: 0.9244 | Regression_eta: 0.0220 | Regression_pt: 1.2977 | Regression_sin_phi: 0.0285 | LR: 4.97e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5778s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0390 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1948 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2684.8 | uss_MB: 1530.5 | child_rss_MB: 22626.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:43:10,036] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:43:10,116] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:43:28,363] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:43:28,363] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:30,731] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:44:30,731] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:01,762] INFO : Step 25000/500000 rank0 | Train Loss: 3.5155 | Train Loss Unweighted: 2.4016 | Train Loss Components: Classification: 0.0042 | Classification_binary: 2.2168 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0862 | Regression_eta: 0.0029 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 2.2168 | Regression_cos_phi: 0.0285 | Regression_energy: 0.4832 | Regression_eta: 0.0171 | Regression_pt: 0.6781 | Regression_sin_phi: 0.0382 | LR: 5.22e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.5537s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0358 | max_loss: 0.0029 | max_backward: 0.0405 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1707 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2682.2 | uss_MB: 1527.9 | child_rss_MB: 22620.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:45:34,085] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:45:34,085] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:36,614] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:46:36,614] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:53,448] INFO : Step 26000/500000 rank0 | Train Loss: 4.8859 | Train Loss Unweighted: 2.8214 | Train Loss Components: Classification: 0.0034 | Classification_binary: 2.4853 | Regression_cos_phi: 0.0225 | Regression_energy: 0.1348 | Regression_eta: 0.0146 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 2.4853 | Regression_cos_phi: 0.2034 | Regression_energy: 0.7557 | Regression_eta: 0.0874 | Regression_pt: 1.1921 | Regression_sin_phi: 0.1186 | LR: 5.47e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.5879s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0359 | max_loss: 0.0028 | max_backward: 0.0392 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.2104 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2683.0 | uss_MB: 1528.7 | child_rss_MB: 22609.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:46:53,450] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:46:53,532] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:47:39,277] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:47:39,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:41,372] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:48:41,372] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:44,524] INFO : Step 27000/500000 rank0 | Train Loss: 3.8627 | Train Loss Unweighted: 2.0752 | Train Loss Components: Classification: 0.0076 | Classification_binary: 1.7813 | Regression_cos_phi: 0.0048 | Regression_energy: 0.1311 | Regression_eta: 0.0086 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0968 | Classification_binary: 1.7813 | Regression_cos_phi: 0.0431 | Regression_energy: 0.7347 | Regression_eta: 0.0513 | Regression_pt: 1.0551 | Regression_sin_phi: 0.1003 | LR: 5.74e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.6031s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0365 | max_loss: 0.0029 | max_backward: 0.0398 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.2191 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2685.0 | uss_MB: 1530.7 | child_rss_MB: 22629.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:49:44,922] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:49:44,922] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:36,910] INFO : Step 28000/500000 rank0 | Train Loss: 2.1607 | Train Loss Unweighted: 1.4832 | Train Loss Components: Classification: 0.0016 | Classification_binary: 1.3716 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0459 | Regression_eta: 0.0067 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0208 | Classification_binary: 1.3716 | Regression_cos_phi: 0.0350 | Regression_energy: 0.2572 | Regression_eta: 0.0402 | Regression_pt: 0.3967 | Regression_sin_phi: 0.0392 | LR: 6.01e-05 | DataLoad Time: 0.0019s | Model Forward Time: 0.5854s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.2000 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2685.2 | uss_MB: 1530.9 | child_rss_MB: 22625.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:50:36,912] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:50:36,988] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:50:48,158] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:50:48,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:50,750] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:51:50,750] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:28,779] INFO : Step 29000/500000 rank0 | Train Loss: 5.0643 | Train Loss Unweighted: 2.4774 | Train Loss Components: Classification: 0.0051 | Classification_binary: 2.0513 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1803 | Regression_eta: 0.0140 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0153 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 2.0513 | Regression_cos_phi: 0.0824 | Regression_energy: 1.0107 | Regression_eta: 0.0835 | Regression_pt: 1.6273 | Regression_sin_phi: 0.1439 | LR: 6.29e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.6029s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0835 | max_tensorboard_logging: 0.2079 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2685.5 | uss_MB: 1531.2 | child_rss_MB: 22619.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:52:53,745] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:52:53,745] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:56,070] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:53:56,076] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:20,773] INFO : Step 30000/500000 rank0 | Train Loss: 4.4491 | Train Loss Unweighted: 2.6262 | Train Loss Components: Classification: 0.0067 | Classification_binary: 2.3300 | Regression_cos_phi: 0.0036 | Regression_energy: 0.1187 | Regression_eta: 0.0063 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0850 | Classification_binary: 2.3300 | Regression_cos_phi: 0.0324 | Regression_energy: 0.6651 | Regression_eta: 0.0376 | Regression_pt: 1.2731 | Regression_sin_phi: 0.0259 | LR: 6.58e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.5909s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.2011 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2682.8 | uss_MB: 1528.5 | child_rss_MB: 22629.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:54:20,775] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:54:20,849] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:54:20,851] INFO : Running validation on rank0 +rank0 [2026-11-14 04:54:20,851] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:54:20,851] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:54:20,976] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:54:21,082] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:54:21,186] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:54:21,255] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:54:21,324] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:54:21,391] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:54:21,458] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:54:21,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:54:21,637] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:54:21,744] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:54:21,850] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:54:21,918] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:54:21,987] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:54:22,054] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:54:22,121] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:54:22,187] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:54:22,287] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:54:22,292] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:54:22,294] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:54:22,296] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:54:22,298] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:54:22,300] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:54:22,303] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:54:22,304] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:54:22,306] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:54:22,308] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:54:22,310] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:54:22,312] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:54:22,315] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:54:22,317] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:54:22,319] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:54:22,321] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:54:23,767] INFO : VALIDATION | Step=30000/500000 | Train Loss=4.4491 | Valid Loss=6.0118 | Train Loss Unweighted=2.6262 | Valid Loss Unweighted=3.1073 | Stale=1 | ETA=879.9m +rank0 [2026-11-14 04:54:23,768] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:54:24,284] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:54:24,296] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:24,347] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:54:24,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:54:25,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:54:25,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:54:25,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:54:25,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:54:26,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:54:26,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:54:26,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:54:26,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:54:27,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:54:27,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:54:27,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:54:27,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:54:28,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:54:28,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:54:28,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:54:28,683] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:54:28,691] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:29,263] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:29,679] INFO : Plotted number of elements +rank0 [2026-11-14 04:54:30,966] INFO : Plotted jets +rank0 [2026-11-14 04:54:31,921] INFO : Plotted jet ratio +rank0 [2026-11-14 04:55:10,899] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:55:10,899] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:13,582] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:56:13,582] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:24,173] INFO : Step 31000/500000 rank0 | Train Loss: 2.8018 | Train Loss Unweighted: 1.4986 | Train Loss Components: Classification: 0.0032 | Classification_binary: 1.2814 | Regression_cos_phi: 0.0096 | Regression_energy: 0.0932 | Regression_eta: 0.0148 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 1.2814 | Regression_cos_phi: 0.0869 | Regression_energy: 0.5226 | Regression_eta: 0.0886 | Regression_pt: 0.7368 | Regression_sin_phi: 0.0446 | LR: 6.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.6123s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0366 | max_loss: 0.0031 | max_backward: 0.0402 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.2201 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2796.7 | uss_MB: 1642.5 | child_rss_MB: 22629.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:57:16,351] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:57:16,352] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:15,279] INFO : Step 32000/500000 rank0 | Train Loss: 3.6138 | Train Loss Unweighted: 1.7641 | Train Loss Components: Classification: 0.0049 | Classification_binary: 1.4598 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1333 | Regression_eta: 0.0227 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0365 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 1.4598 | Regression_cos_phi: 0.0505 | Regression_energy: 0.7472 | Regression_eta: 0.1357 | Regression_pt: 0.8147 | Regression_sin_phi: 0.3430 | LR: 7.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5460s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1531 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2796.7 | uss_MB: 1642.5 | child_rss_MB: 22627.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:58:15,281] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:58:15,363] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:58:19,202] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:58:19,202] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:21,739] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:59:21,739] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:06,009] INFO : Step 33000/500000 rank0 | Train Loss: 2.7133 | Train Loss Unweighted: 1.6880 | Train Loss Components: Classification: 0.0112 | Classification_binary: 1.5263 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0677 | Regression_eta: 0.0054 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1427 | Classification_binary: 1.5263 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3797 | Regression_eta: 0.0321 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0335 | LR: 7.51e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5557s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0386 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1711 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2707.4 | uss_MB: 1553.1 | child_rss_MB: 22625.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:00:23,532] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:00:23,532] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:26,944] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:01:26,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:00,347] INFO : Step 34000/500000 rank0 | Train Loss: 3.6205 | Train Loss Unweighted: 1.8775 | Train Loss Components: Classification: 0.0061 | Classification_binary: 1.5864 | Regression_cos_phi: 0.0032 | Regression_energy: 0.1347 | Regression_eta: 0.0071 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 1.5864 | Regression_cos_phi: 0.0290 | Regression_energy: 0.7548 | Regression_eta: 0.0421 | Regression_pt: 1.1065 | Regression_sin_phi: 0.0232 | LR: 7.83e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.6301s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0031 | max_backward: 0.0417 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.2231 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2707.9 | uss_MB: 1553.6 | child_rss_MB: 22610.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:02:00,350] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:02:00,431] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:02:32,565] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:02:32,566] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:34,951] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:03:34,951] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:53,198] INFO : Step 35000/500000 rank0 | Train Loss: 2.8437 | Train Loss Unweighted: 2.0443 | Train Loss Components: Classification: 0.0083 | Classification_binary: 1.9195 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0493 | Regression_eta: 0.0046 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1061 | Classification_binary: 1.9195 | Regression_cos_phi: 0.0428 | Regression_energy: 0.2761 | Regression_eta: 0.0276 | Regression_pt: 0.4323 | Regression_sin_phi: 0.0394 | LR: 8.16e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.5785s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1899 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2709.9 | uss_MB: 1555.6 | child_rss_MB: 22630.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:04:38,050] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:04:38,050] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:40,357] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:05:40,358] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:44,318] INFO : Step 36000/500000 rank0 | Train Loss: 2.2937 | Train Loss Unweighted: 1.1014 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.9001 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0921 | Regression_eta: 0.0073 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0181 | Classification_binary: 0.9001 | Regression_cos_phi: 0.0183 | Regression_energy: 0.5160 | Regression_eta: 0.0433 | Regression_pt: 0.7622 | Regression_sin_phi: 0.0358 | LR: 8.50e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.5436s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0368 | max_loss: 0.0031 | max_backward: 0.0404 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1448 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2709.9 | uss_MB: 1555.7 | child_rss_MB: 22628.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 05:05:44,320] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:05:44,397] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:06:42,837] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:06:42,837] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:35,441] INFO : Step 37000/500000 rank0 | Train Loss: 1.5534 | Train Loss Unweighted: 0.6330 | Train Loss Components: Classification: 0.0015 | Classification_binary: 0.4819 | Regression_cos_phi: 0.0033 | Regression_energy: 0.0577 | Regression_eta: 0.0134 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0196 | Classification_binary: 0.4819 | Regression_cos_phi: 0.0302 | Regression_energy: 0.3233 | Regression_eta: 0.0800 | Regression_pt: 0.5208 | Regression_sin_phi: 0.0975 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5582s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0358 | max_loss: 0.0029 | max_backward: 0.0401 | max_optimizer_step: 0.0779 | max_tensorboard_logging: 0.1666 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2710.0 | uss_MB: 1555.7 | child_rss_MB: 22627.7 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:07:45,515] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:07:45,515] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:47,845] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:08:47,845] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:27,033] INFO : Step 38000/500000 rank0 | Train Loss: 2.5373 | Train Loss Unweighted: 1.0192 | Train Loss Components: Classification: 0.0022 | Classification_binary: 0.7626 | Regression_cos_phi: 0.0035 | Regression_energy: 0.1217 | Regression_eta: 0.0047 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.7626 | Regression_cos_phi: 0.0312 | Regression_energy: 0.6824 | Regression_eta: 0.0281 | Regression_pt: 0.9812 | Regression_sin_phi: 0.0234 | LR: 9.21e-05 | DataLoad Time: 0.0022s | Model Forward Time: 0.5785s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0404 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1866 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2707.9 | uss_MB: 1553.7 | child_rss_MB: 22622.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:09:27,035] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:09:27,116] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:09:51,383] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:09:51,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:53,817] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:10:53,817] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:18,938] INFO : Step 39000/500000 rank0 | Train Loss: 2.6553 | Train Loss Unweighted: 1.3425 | Train Loss Components: Classification: 0.0207 | Classification_binary: 1.1429 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0814 | Regression_eta: 0.0047 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2641 | Classification_binary: 1.1429 | Regression_cos_phi: 0.0625 | Regression_energy: 0.4561 | Regression_eta: 0.0280 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0777 | LR: 9.57e-05 | DataLoad Time: 0.0023s | Model Forward Time: 0.5742s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0385 | max_loss: 0.0032 | max_backward: 0.0395 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1917 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2710.0 | uss_MB: 1555.8 | child_rss_MB: 22629.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:11:56,849] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:11:56,849] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:59,914] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:12:59,915] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:11,398] INFO : Step 40000/500000 rank0 | Train Loss: 1.5230 | Train Loss Unweighted: 0.6471 | Train Loss Components: Classification: 0.0025 | Classification_binary: 0.5007 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0667 | Regression_eta: 0.0075 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0322 | Classification_binary: 0.5007 | Regression_cos_phi: 0.0260 | Regression_energy: 0.3740 | Regression_eta: 0.0446 | Regression_pt: 0.4911 | Regression_sin_phi: 0.0544 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5708s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0408 | max_loss: 0.0033 | max_backward: 0.0420 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1836 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2710.4 | uss_MB: 1556.1 | child_rss_MB: 22629.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:13:11,401] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:13:11,474] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:13:11,476] INFO : Running validation on rank0 +rank0 [2026-11-14 05:13:11,476] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:13:11,476] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:13:11,604] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:13:11,710] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:13:11,810] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:13:11,879] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:13:11,945] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:13:12,011] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:13:12,080] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:13:12,147] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:13:12,261] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:13:12,371] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:13:12,481] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:13:12,550] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:13:12,620] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:13:12,687] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:13:12,755] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:13:12,823] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:13:12,934] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:13:12,940] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:13:12,943] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:13:12,944] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:13:12,947] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:13:12,949] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:13:12,951] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:13:12,952] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:13:12,954] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:13:12,956] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:13:12,959] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:13:12,961] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:13:12,963] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:13:12,964] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:13:12,966] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:13:12,968] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:13:14,433] INFO : VALIDATION | Step=40000/500000 | Train Loss=1.5230 | Valid Loss=8.1474 | Train Loss Unweighted=0.6471 | Valid Loss Unweighted=4.6317 | Stale=2 | ETA=862.6m +rank0 [2026-11-14 05:13:14,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:13:15,012] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:13:15,027] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:15,076] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:13:15,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:13:15,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:13:16,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:13:16,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:13:16,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:13:16,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:13:17,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:13:17,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:13:17,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:13:17,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:13:18,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:13:18,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:13:18,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:13:18,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:13:19,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:13:19,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:13:19,369] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:13:19,377] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:19,924] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:20,341] INFO : Plotted number of elements +rank0 [2026-11-14 05:13:21,631] INFO : Plotted jets +rank0 [2026-11-14 05:13:22,577] INFO : Plotted jet ratio +rank0 [2026-11-14 05:14:14,663] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:14:14,663] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:14,438] INFO : Step 41000/500000 rank0 | Train Loss: 2.9571 | Train Loss Unweighted: 1.3747 | Train Loss Components: Classification: 0.0039 | Classification_binary: 1.1072 | Regression_cos_phi: 0.0018 | Regression_energy: 0.1289 | Regression_eta: 0.0048 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 1.1072 | Regression_cos_phi: 0.0164 | Regression_energy: 0.7226 | Regression_eta: 0.0287 | Regression_pt: 1.0150 | Regression_sin_phi: 0.0172 | LR: 1.03e-04 | DataLoad Time: 0.0015s | Model Forward Time: 0.5375s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0398 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1528 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2733.2 | uss_MB: 1578.9 | child_rss_MB: 22627.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:15:17,304] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:15:17,305] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:19,484] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:16:19,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:05,598] INFO : Step 42000/500000 rank0 | Train Loss: 1.2687 | Train Loss Unweighted: 0.5117 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.3862 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0535 | Regression_eta: 0.0052 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0070 | Classification_binary: 0.3862 | Regression_cos_phi: 0.0378 | Regression_energy: 0.3001 | Regression_eta: 0.0313 | Regression_pt: 0.4564 | Regression_sin_phi: 0.0500 | LR: 1.07e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.5586s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0453 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1762 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2733.6 | uss_MB: 1579.4 | child_rss_MB: 22616.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:17:05,600] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:17:05,680] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:17:22,366] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:17:22,366] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:25,088] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:18:25,088] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:57,475] INFO : Step 43000/500000 rank0 | Train Loss: 1.7196 | Train Loss Unweighted: 0.7058 | Train Loss Components: Classification: 0.0075 | Classification_binary: 0.5398 | Regression_cos_phi: 0.0065 | Regression_energy: 0.0746 | Regression_eta: 0.0115 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0953 | Classification_binary: 0.5398 | Regression_cos_phi: 0.0592 | Regression_energy: 0.4184 | Regression_eta: 0.0689 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0656 | LR: 1.11e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5873s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0031 | max_backward: 0.0401 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.2016 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2735.4 | uss_MB: 1581.2 | child_rss_MB: 22613.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:19:27,976] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:19:27,977] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:30,266] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:20:30,266] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:48,709] INFO : Step 44000/500000 rank0 | Train Loss: 1.3215 | Train Loss Unweighted: 0.5040 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.3690 | Regression_cos_phi: 0.0036 | Regression_energy: 0.0540 | Regression_eta: 0.0089 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0033 | Classification_binary: 0.3690 | Regression_cos_phi: 0.0327 | Regression_energy: 0.3026 | Regression_eta: 0.0533 | Regression_pt: 0.4780 | Regression_sin_phi: 0.0826 | LR: 1.15e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5258s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1470 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2736.4 | uss_MB: 1582.1 | child_rss_MB: 22630.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:20:48,711] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:20:48,785] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:21:33,011] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:21:33,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:35,460] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:22:35,461] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:40,690] INFO : Step 45000/500000 rank0 | Train Loss: 1.0263 | Train Loss Unweighted: 0.4690 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.3752 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0412 | Regression_eta: 0.0051 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0012 | Classification_binary: 0.3752 | Regression_cos_phi: 0.0274 | Regression_energy: 0.2310 | Regression_eta: 0.0304 | Regression_pt: 0.3295 | Regression_sin_phi: 0.0315 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.5603s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1760 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2736.4 | uss_MB: 1582.2 | child_rss_MB: 22629.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:23:39,110] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:23:39,110] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:34,700] INFO : Step 46000/500000 rank0 | Train Loss: 0.8002 | Train Loss Unweighted: 0.2205 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.1227 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0429 | Regression_eta: 0.0060 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0027 | Classification_binary: 0.1227 | Regression_cos_phi: 0.0256 | Regression_energy: 0.2403 | Regression_eta: 0.0359 | Regression_pt: 0.3469 | Regression_sin_phi: 0.0259 | LR: 1.23e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.5604s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0395 | max_optimizer_step: 0.0823 | max_tensorboard_logging: 0.1622 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2734.0 | uss_MB: 1579.8 | child_rss_MB: 22629.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:24:34,702] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 05:24:34,780] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:24:44,109] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 05:24:44,109] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:45,844] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 05:25:45,844] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:26,032] INFO : Step 47000/500000 rank0 | Train Loss: 1.1431 | Train Loss Unweighted: 0.4855 | Train Loss Components: Classification: 0.0094 | Classification_binary: 0.3848 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0372 | Regression_eta: 0.0068 | Regression_pt: 0.0398 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1198 | Classification_binary: 0.3848 | Regression_cos_phi: 0.0276 | Regression_energy: 0.2086 | Regression_eta: 0.0406 | Regression_pt: 0.3201 | Regression_sin_phi: 0.0414 | LR: 1.27e-04 | DataLoad Time: 0.0023s | Model Forward Time: 0.6016s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.2078 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2735.8 | uss_MB: 1581.6 | child_rss_MB: 22624.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:26:49,321] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 05:26:49,321] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:52,350] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 05:27:52,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:18,478] INFO : Step 48000/500000 rank0 | Train Loss: 0.9457 | Train Loss Unweighted: 0.2672 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0161 | Regression_energy: 0.0366 | Regression_eta: 0.0048 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0232 | Classification_binary: 0.1610 | Regression_cos_phi: 0.1452 | Regression_energy: 0.2052 | Regression_eta: 0.0288 | Regression_pt: 0.3446 | Regression_sin_phi: 0.0376 | LR: 1.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.6345s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.2486 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2735.8 | uss_MB: 1581.6 | child_rss_MB: 22619.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:28:18,480] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 05:28:18,555] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:28:55,525] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:28:55,525] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:29:57,707] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:29:57,707] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:09,930] INFO : Step 49000/500000 rank0 | Train Loss: 1.9961 | Train Loss Unweighted: 1.4607 | Train Loss Components: Classification: 0.0006 | Classification_binary: 1.3725 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0357 | Regression_eta: 0.0069 | Regression_pt: 0.0344 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0080 | Classification_binary: 1.3725 | Regression_cos_phi: 0.0448 | Regression_energy: 0.2002 | Regression_eta: 0.0411 | Regression_pt: 0.2767 | Regression_sin_phi: 0.0529 | LR: 1.36e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5311s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0386 | max_loss: 0.0030 | max_backward: 0.0390 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1490 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2735.8 | uss_MB: 1581.6 | child_rss_MB: 22627.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:31:00,378] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:31:00,378] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:01,583] INFO : Step 50000/500000 rank0 | Train Loss: 1.2646 | Train Loss Unweighted: 0.7548 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.6693 | Regression_cos_phi: 0.0015 | Regression_energy: 0.0388 | Regression_eta: 0.0017 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0048 | Classification_binary: 0.6693 | Regression_cos_phi: 0.0137 | Regression_energy: 0.2175 | Regression_eta: 0.0102 | Regression_pt: 0.3340 | Regression_sin_phi: 0.0151 | LR: 1.40e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.5900s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0391 | max_optimizer_step: 0.0871 | max_tensorboard_logging: 0.1869 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2733.4 | uss_MB: 1579.1 | child_rss_MB: 22622.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:32:01,585] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:32:01,668] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:32:01,671] INFO : Running validation on rank0 +rank0 [2026-11-14 05:32:01,671] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:32:01,671] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:32:01,795] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:32:01,898] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:32:01,998] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:32:02,068] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:32:02,134] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:32:02,199] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:32:02,265] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:32:02,332] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:32:02,443] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:32:02,550] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:32:02,655] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:32:02,723] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:32:02,789] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:32:02,857] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:32:02,925] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:32:02,994] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:32:03,105] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:32:03,110] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:32:03,112] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:32:03,115] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:32:03,117] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:32:03,120] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:32:03,122] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:32:03,124] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:32:03,126] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:32:03,129] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:32:03,131] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:32:03,133] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:32:03,136] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:32:03,137] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:32:03,139] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:32:03,141] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:32:04,589] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.2646 | Valid Loss=10.2431 | Train Loss Unweighted=0.7548 | Valid Loss Unweighted=6.7029 | Stale=3 | ETA=844.6m +rank0 [2026-11-14 05:32:04,589] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:32:05,057] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:32:05,072] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:05,123] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:32:05,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:32:05,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:32:06,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:32:06,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:32:06,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:32:06,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:32:07,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:32:07,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:32:07,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:32:07,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:32:08,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:32:08,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:32:08,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:32:08,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:32:09,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:32:09,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:32:09,368] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:32:09,375] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:09,915] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:10,315] INFO : Plotted number of elements +rank0 [2026-11-14 05:32:11,582] INFO : Plotted jets +rank0 [2026-11-14 05:32:12,505] INFO : Plotted jet ratio +rank0 [2026-11-14 05:32:12,911] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:32:12,911] INFO : Training completed. Total time on device 0: 93.985min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..2994976a1c8555173878242f57b901489b5da1a3 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/train.log.0 @@ -0,0 +1,997 @@ +rank0 [2026-11-14 03:58:08,100] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 03:58:08,100] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 03:58:08,102] INFO : Instantiating model +rank0 [2026-11-14 03:58:08,136] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 03:58:08,136] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 03:58:08,138] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 03:58:08,138] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 03:58:08,138] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 03:58:08,138] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 03:58:08,138] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,140] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 03:58:08,140] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,143] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 03:58:08,143] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,146] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 03:58:08,146] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,149] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 03:58:08,149] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,151] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 03:58:08,151] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,154] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 03:58:08,154] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 03:58:08,154] INFO : Initializing output DNNs +rank0 [2026-11-14 03:58:08,167] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,169] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:08,177] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-14 03:58:08,177] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 03:58:08,177] INFO : jagged_attention enabled=True +rank0 [2026-11-14 03:58:08,177] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 03:58:08,177] INFO : task_queries enabled=True +rank0 [2026-11-14 03:58:08,177] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 03:58:08,177] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 03:58:08,177] INFO : classification_query parameters: 256 +rank0 [2026-11-14 03:58:08,177] INFO : regression_query parameters: 256 +rank0 [2026-11-14 03:58:08,177] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 03:58:08,177] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 03:58:08,177] INFO : output_mode=elementwise +rank0 [2026-11-14 03:58:08,177] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 03:58:08,177] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 03:58:08,177] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 03:58:08,177] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 03:58:08,177] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 03:58:08,178] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 03:58:08,178] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 03:58:08,178] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 03:58:08,179] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 03:58:08,179] INFO : MLPF __init__ done +rank0 [2026-11-14 03:58:08,179] INFO : Instantiated model +rank0 [2026-11-14 03:58:08,180] INFO : Moving model to device rank=0 +rank0 [2026-11-14 03:58:08,194] INFO : Moved model to device rank=0 +rank0 [2026-11-14 03:58:08,195] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 03:58:08,309] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 03:58:08,309] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 03:58:08,311] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 03:58:08,311] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 03:58:08,311] INFO : Total parameters: 3910930 +rank0 [2026-11-14 03:58:08,313] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 03:58:08,319] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077 +rank0 [2026-11-14 03:58:08,319] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077 +rank0 [2026-11-14 03:58:08,319] INFO : Getting dataloaders +rank0 [2026-11-14 03:58:08,319] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:58:12,974] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:13,025] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:13,081] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:13,190] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,243] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,314] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,364] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,418] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,465] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:13,517] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:13,517] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 03:58:13,518] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 03:58:13,518] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 03:58:13,518] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:58:13,529] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,538] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,554] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,571] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,581] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,599] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,609] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,621] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,629] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,640] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:13,640] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 03:58:13,640] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 03:58:13,713] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:58:13,826] INFO : Got dataloaders +rank0 [2026-11-14 03:58:13,826] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 03:58:13,826] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 03:58:13,827] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 03:58:13,834] INFO : Creating train_iterator +rank0 [2026-11-14 03:58:13,834] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 03:58:13,861] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:58:14,005] INFO : Created train_iterator +rank0 [2026-11-14 03:58:27,169] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.600406875610352, 'Classification': 0.5960752093791961, 'Regression_pt': 0.9448277205228806, 'Regression_eta': 1.273502309024334, 'Regression_sin_phi': 0.8084679882228375, 'Regression_cos_phi': 0.840230675637722, 'Regression_energy': 1.356031168103218}, weights={'Classification_binary': 1.0, 'Classification': 12.750751495361328, 'Regression_pt': 8.044224739074707, 'Regression_eta': 5.968113899230957, 'Regression_sin_phi': 9.400999069213867, 'Regression_cos_phi': 9.045619010925293, 'Regression_energy': 5.604890823364258} +rank0 [2026-11-14 03:59:19,013] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 03:59:19,014] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:00:08,136] INFO : Step 1000/500000 rank0 | Train Loss: 16.5767 | Train Loss Unweighted: 6.0048 | Train Loss Components: Classification: 0.0973 | Classification_binary: 4.3464 | Regression_cos_phi: 0.0407 | Regression_energy: 0.6390 | Regression_eta: 0.0493 | Regression_pt: 0.7932 | Regression_sin_phi: 0.0388 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 1.2403 | Classification_binary: 4.3464 | Regression_cos_phi: 0.3683 | Regression_energy: 3.5816 | Regression_eta: 0.2941 | Regression_pt: 6.3808 | Regression_sin_phi: 0.3651 | LR: 2.01e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5777s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0355 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1942 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2566.9 | uss_MB: 1398.6 | child_rss_MB: 22726.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:00:21,818] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:00:21,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:23,884] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:01:23,884] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:58,936] INFO : Step 2000/500000 rank0 | Train Loss: 9.5641 | Train Loss Unweighted: 4.1708 | Train Loss Components: Classification: 0.0381 | Classification_binary: 3.2961 | Regression_cos_phi: 0.0152 | Regression_energy: 0.3862 | Regression_eta: 0.0223 | Regression_pt: 0.3942 | Regression_sin_phi: 0.0186 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.4863 | Classification_binary: 3.2961 | Regression_cos_phi: 0.1378 | Regression_energy: 2.1648 | Regression_eta: 0.1332 | Regression_pt: 3.1708 | Regression_sin_phi: 0.1752 | LR: 2.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5858s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0397 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.2122 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2567.6 | uss_MB: 1399.4 | child_rss_MB: 22629.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:01:58,939] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:02:26,071] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:02:26,072] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:28,005] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:03:28,005] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:49,207] INFO : Step 3000/500000 rank0 | Train Loss: 7.4587 | Train Loss Unweighted: 3.9207 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.3561 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2471 | Regression_eta: 0.0117 | Regression_pt: 0.2485 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.4327 | Classification_binary: 3.3561 | Regression_cos_phi: 0.0866 | Regression_energy: 1.3850 | Regression_eta: 0.0699 | Regression_pt: 1.9991 | Regression_sin_phi: 0.1293 | LR: 2.05e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5567s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1906 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.5 | uss_MB: 1402.7 | child_rss_MB: 22629.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:04:30,345] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:04:30,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:32,209] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:05:32,209] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:40,003] INFO : Step 4000/500000 rank0 | Train Loss: 7.2400 | Train Loss Unweighted: 3.2831 | Train Loss Components: Classification: 0.0523 | Classification_binary: 2.6559 | Regression_cos_phi: 0.0255 | Regression_energy: 0.2617 | Regression_eta: 0.0547 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.6665 | Classification_binary: 2.6559 | Regression_cos_phi: 0.2305 | Regression_energy: 1.4670 | Regression_eta: 0.3267 | Regression_pt: 1.7591 | Regression_sin_phi: 0.1344 | LR: 2.08e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.5717s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0396 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1974 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2570.5 | uss_MB: 1402.7 | child_rss_MB: 22628.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:05:40,005] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:06:35,060] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:06:35,060] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:30,921] INFO : Step 5000/500000 rank0 | Train Loss: 2.9962 | Train Loss Unweighted: 2.0725 | Train Loss Components: Classification: 0.0217 | Classification_binary: 1.9383 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0490 | Regression_eta: 0.0076 | Regression_pt: 0.0455 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2766 | Classification_binary: 1.9383 | Regression_cos_phi: 0.0620 | Regression_energy: 0.2744 | Regression_eta: 0.0453 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0338 | LR: 2.13e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5818s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0400 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.2028 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2568.8 | uss_MB: 1401.0 | child_rss_MB: 22626.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:07:37,404] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:07:37,404] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:39,933] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:08:39,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:21,273] INFO : Step 6000/500000 rank0 | Train Loss: 8.1058 | Train Loss Unweighted: 3.3835 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5957 | Regression_cos_phi: 0.0069 | Regression_energy: 0.3743 | Regression_eta: 0.0129 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2672 | Classification_binary: 2.5957 | Regression_cos_phi: 0.0626 | Regression_energy: 2.0979 | Regression_eta: 0.0771 | Regression_pt: 2.9566 | Regression_sin_phi: 0.0487 | LR: 2.19e-05 | DataLoad Time: 0.0025s | Model Forward Time: 0.5605s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0028 | max_backward: 0.0411 | max_optimizer_step: 0.0641 | max_tensorboard_logging: 0.2076 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.8 | uss_MB: 1401.0 | child_rss_MB: 22625.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:09:21,275] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:09:41,160] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:09:41,160] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:42,963] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:10:42,963] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:11,897] INFO : Step 7000/500000 rank0 | Train Loss: 6.8405 | Train Loss Unweighted: 3.6048 | Train Loss Components: Classification: 0.0153 | Classification_binary: 3.0728 | Regression_cos_phi: 0.0117 | Regression_energy: 0.2333 | Regression_eta: 0.0205 | Regression_pt: 0.2393 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1957 | Classification_binary: 3.0728 | Regression_cos_phi: 0.1059 | Regression_energy: 1.3077 | Regression_eta: 0.1221 | Regression_pt: 1.9252 | Regression_sin_phi: 0.1111 | LR: 2.26e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5588s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1860 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2569.8 | uss_MB: 1402.0 | child_rss_MB: 22621.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:11:45,477] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:11:45,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:12:47,693] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:12:47,693] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:02,786] INFO : Step 8000/500000 rank0 | Train Loss: 6.8290 | Train Loss Unweighted: 4.2079 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.7898 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1823 | Regression_eta: 0.0076 | Regression_pt: 0.1880 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.3238 | Classification_binary: 3.7898 | Regression_cos_phi: 0.0715 | Regression_energy: 1.0218 | Regression_eta: 0.0454 | Regression_pt: 1.5124 | Regression_sin_phi: 0.0643 | LR: 2.34e-05 | DataLoad Time: 0.0019s | Model Forward Time: 0.5877s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0355 | max_loss: 0.0028 | max_backward: 0.0392 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1980 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2569.9 | uss_MB: 1402.1 | child_rss_MB: 22618.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:13:02,788] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:13:02,867] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:13:50,206] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:13:50,207] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:52,203] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:14:52,203] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:53,534] INFO : Step 9000/500000 rank0 | Train Loss: 7.3855 | Train Loss Unweighted: 3.2611 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5667 | Regression_cos_phi: 0.0089 | Regression_energy: 0.2656 | Regression_eta: 0.1095 | Regression_pt: 0.2830 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2350 | Classification_binary: 2.5667 | Regression_cos_phi: 0.0808 | Regression_energy: 1.4886 | Regression_eta: 0.6534 | Regression_pt: 2.2765 | Regression_sin_phi: 0.0844 | LR: 2.43e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5319s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1634 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2567.9 | uss_MB: 1400.1 | child_rss_MB: 22599.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:15:54,315] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:15:54,315] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:16:44,313] INFO : Step 10000/500000 rank0 | Train Loss: 6.3936 | Train Loss Unweighted: 3.0321 | Train Loss Components: Classification: 0.0150 | Classification_binary: 2.4704 | Regression_cos_phi: 0.0040 | Regression_energy: 0.2714 | Regression_eta: 0.0066 | Regression_pt: 0.2601 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1911 | Classification_binary: 2.4704 | Regression_cos_phi: 0.0362 | Regression_energy: 1.5210 | Regression_eta: 0.0395 | Regression_pt: 2.0920 | Regression_sin_phi: 0.0434 | LR: 2.52e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.6015s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0027 | max_backward: 0.0387 | max_optimizer_step: 0.0661 | max_tensorboard_logging: 0.2324 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.5 | uss_MB: 1400.6 | child_rss_MB: 22621.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:16:44,315] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:16:44,391] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:16:44,392] INFO : Running validation on rank0 +rank0 [2026-11-14 04:16:44,393] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:16:44,393] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:16:44,634] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:16:44,913] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:16:44,982] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:16:45,036] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:16:45,090] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:16:45,136] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:16:45,190] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:16:45,243] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:16:45,293] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:16:45,383] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:16:45,455] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:16:45,509] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:16:45,563] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:16:45,611] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:16:45,664] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:16:45,719] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:16:45,771] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:16:45,774] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:16:45,777] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:16:45,779] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:16:45,781] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:16:45,784] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:16:45,786] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:16:45,788] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:16:45,790] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:16:45,792] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:16:45,794] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:16:45,797] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:16:45,799] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:16:45,801] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:16:45,803] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:16:45,806] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:16:47,307] INFO : VALIDATION | Step=10000/500000 | Train Loss=6.3936 | Valid Loss=5.7690 | Train Loss Unweighted=3.0321 | Valid Loss Unweighted=2.9586 | Stale=0 | ETA=909.3m +rank0 [2026-11-14 04:16:47,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:16:47,902] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:16:47,916] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:16:47,973] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:16:49,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:16:49,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:16:49,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:16:49,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:16:50,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:16:50,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:16:50,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:16:50,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:16:51,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:16:51,316] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:16:51,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:16:51,818] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:16:52,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:16:52,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:16:52,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:16:52,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:16:52,881] INFO : Time taken to make predictions on device 0 is: 0.08 min +rank0 [2026-11-14 04:16:52,891] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:16:53,446] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:16:55,023] INFO : Plotted number of elements +rank0 [2026-11-14 04:16:56,364] INFO : Plotted jets +rank0 [2026-11-14 04:16:57,316] INFO : Plotted jet ratio +rank0 [2026-11-14 04:17:10,700] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:17:10,701] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:12,232] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:18:12,232] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:48,219] INFO : Step 11000/500000 rank0 | Train Loss: 6.0989 | Train Loss Unweighted: 2.7872 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.2435 | Regression_cos_phi: 0.0110 | Regression_energy: 0.2509 | Regression_eta: 0.0074 | Regression_pt: 0.2500 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2463 | Classification_binary: 2.2435 | Regression_cos_phi: 0.0999 | Regression_energy: 1.4062 | Regression_eta: 0.0440 | Regression_pt: 2.0108 | Regression_sin_phi: 0.0482 | LR: 2.63e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.6180s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.2368 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2706.0 | uss_MB: 1551.6 | child_rss_MB: 22630.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:19:14,923] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:19:14,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:17,112] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:20:17,112] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:39,615] INFO : Step 12000/500000 rank0 | Train Loss: 2.9827 | Train Loss Unweighted: 2.1655 | Train Loss Components: Classification: 0.0037 | Classification_binary: 2.0321 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0577 | Regression_eta: 0.0041 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 2.0321 | Regression_cos_phi: 0.0399 | Regression_energy: 0.3234 | Regression_eta: 0.0243 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0361 | LR: 2.75e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.5541s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1695 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2706.0 | uss_MB: 1551.6 | child_rss_MB: 22629.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:20:39,617] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:20:39,696] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:21:20,128] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:21:20,128] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:23,031] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:22:23,031] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:31,732] INFO : Step 13000/500000 rank0 | Train Loss: 5.1274 | Train Loss Unweighted: 2.9942 | Train Loss Components: Classification: 0.0308 | Classification_binary: 2.6677 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1261 | Regression_eta: 0.0097 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.3924 | Classification_binary: 2.6677 | Regression_cos_phi: 0.0711 | Regression_energy: 0.7067 | Regression_eta: 0.0579 | Regression_pt: 1.1743 | Regression_sin_phi: 0.0571 | LR: 2.88e-05 | DataLoad Time: 0.0024s | Model Forward Time: 0.5520s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0396 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1688 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2650.1 | uss_MB: 1495.8 | child_rss_MB: 22627.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:23:25,886] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:23:25,886] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:23,495] INFO : Step 14000/500000 rank0 | Train Loss: 5.4227 | Train Loss Unweighted: 3.3421 | Train Loss Components: Classification: 0.0261 | Classification_binary: 3.0139 | Regression_cos_phi: 0.0042 | Regression_energy: 0.1432 | Regression_eta: 0.0075 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.3332 | Classification_binary: 3.0139 | Regression_cos_phi: 0.0380 | Regression_energy: 0.8027 | Regression_eta: 0.0450 | Regression_pt: 1.1487 | Regression_sin_phi: 0.0412 | LR: 3.02e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5849s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0363 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.2045 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2650.6 | uss_MB: 1496.3 | child_rss_MB: 22626.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:23,497] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:24:23,580] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:24:29,170] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:24:29,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:31,600] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:25:31,601] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:15,244] INFO : Step 15000/500000 rank0 | Train Loss: 4.6705 | Train Loss Unweighted: 2.9439 | Train Loss Components: Classification: 0.0153 | Classification_binary: 2.6712 | Regression_cos_phi: 0.0039 | Regression_energy: 0.1090 | Regression_eta: 0.0057 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1949 | Classification_binary: 2.6712 | Regression_cos_phi: 0.0352 | Regression_energy: 0.6110 | Regression_eta: 0.0343 | Regression_pt: 1.0727 | Regression_sin_phi: 0.0512 | LR: 3.17e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5555s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0355 | max_loss: 0.0028 | max_backward: 0.0452 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1744 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2654.6 | uss_MB: 1500.2 | child_rss_MB: 22626.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:26:34,450] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:26:34,450] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:36,896] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:27:36,896] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:06,780] INFO : Step 16000/500000 rank0 | Train Loss: 6.2304 | Train Loss Unweighted: 3.8409 | Train Loss Components: Classification: 0.0140 | Classification_binary: 3.4472 | Regression_cos_phi: 0.0043 | Regression_energy: 0.1806 | Regression_eta: 0.0114 | Regression_pt: 0.1760 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1780 | Classification_binary: 3.4472 | Regression_cos_phi: 0.0393 | Regression_energy: 1.0124 | Regression_eta: 0.0680 | Regression_pt: 1.4160 | Regression_sin_phi: 0.0696 | LR: 3.33e-05 | DataLoad Time: 0.0025s | Model Forward Time: 0.6000s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0029 | max_backward: 0.0406 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.2006 | max_tensorboard_flush: 0.0007 | Memory: rss_MB: 2654.7 | uss_MB: 1500.4 | child_rss_MB: 22622.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:28:06,782] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:06,855] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:28:39,767] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:28:39,767] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:42,004] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:29:42,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:58,125] INFO : Step 17000/500000 rank0 | Train Loss: 4.3197 | Train Loss Unweighted: 2.8574 | Train Loss Components: Classification: 0.0048 | Classification_binary: 2.6128 | Regression_cos_phi: 0.0026 | Regression_energy: 0.1149 | Regression_eta: 0.0058 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 2.6128 | Regression_cos_phi: 0.0232 | Regression_energy: 0.6438 | Regression_eta: 0.0349 | Regression_pt: 0.9030 | Regression_sin_phi: 0.0403 | LR: 3.51e-05 | DataLoad Time: 0.0018s | Model Forward Time: 0.5980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0396 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2649.6 | uss_MB: 1495.3 | child_rss_MB: 22610.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:30:45,077] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:30:45,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:47,327] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:31:47,327] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:49,508] INFO : Step 18000/500000 rank0 | Train Loss: 4.8641 | Train Loss Unweighted: 3.0079 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7212 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1058 | Regression_eta: 0.0114 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2721 | Classification_binary: 2.7212 | Regression_cos_phi: 0.0907 | Regression_energy: 0.5933 | Regression_eta: 0.0680 | Regression_pt: 1.0601 | Regression_sin_phi: 0.0588 | LR: 3.69e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5300s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0032 | max_backward: 0.0394 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1605 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2650.6 | uss_MB: 1496.2 | child_rss_MB: 22627.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:31:49,510] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:31:49,590] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:32:49,520] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:32:49,521] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:41,007] INFO : Step 19000/500000 rank0 | Train Loss: 3.3378 | Train Loss Unweighted: 1.7679 | Train Loss Components: Classification: 0.0055 | Classification_binary: 1.4958 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1365 | Regression_eta: 0.0240 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 1.4958 | Regression_cos_phi: 0.0410 | Regression_energy: 0.7648 | Regression_eta: 0.1434 | Regression_pt: 0.7836 | Regression_sin_phi: 0.0388 | LR: 3.88e-05 | DataLoad Time: 0.0021s | Model Forward Time: 0.6656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0029 | max_backward: 0.0426 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2654.4 | uss_MB: 1500.1 | child_rss_MB: 22623.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:33:53,006] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:33:53,006] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:54,233] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:34:54,233] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:31,556] INFO : Step 20000/500000 rank0 | Train Loss: 4.6341 | Train Loss Unweighted: 2.3064 | Train Loss Components: Classification: 0.0028 | Classification_binary: 1.9142 | Regression_cos_phi: 0.0048 | Regression_energy: 0.1851 | Regression_eta: 0.0067 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 1.9142 | Regression_cos_phi: 0.0438 | Regression_energy: 1.0373 | Regression_eta: 0.0403 | Regression_pt: 1.4778 | Regression_sin_phi: 0.0850 | LR: 4.07e-05 | DataLoad Time: 0.0021s | Model Forward Time: 0.6126s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0032 | max_backward: 0.0396 | max_optimizer_step: 0.0769 | max_tensorboard_logging: 0.2090 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2654.8 | uss_MB: 1500.5 | child_rss_MB: 22657.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:31,558] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:35:31,640] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:35:31,642] INFO : Running validation on rank0 +rank0 [2026-11-14 04:35:31,642] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:35:31,642] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:35:31,768] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:35:31,876] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:35:31,985] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:35:32,057] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:35:32,126] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:35:32,194] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:35:32,262] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:35:32,331] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:35:32,437] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:35:32,545] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:35:32,652] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:35:32,721] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:35:32,788] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:35:32,855] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:35:32,926] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:35:32,996] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:35:33,113] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:35:33,118] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:35:33,121] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:35:33,124] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:35:33,126] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:35:33,128] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:35:33,130] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:35:33,132] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:35:33,134] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:35:33,136] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:35:33,137] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:35:33,139] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:35:33,141] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:35:33,144] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:35:33,147] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:35:33,148] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:35:34,698] INFO : VALIDATION | Step=20000/500000 | Train Loss=4.6341 | Valid Loss=5.5607 | Train Loss Unweighted=2.3064 | Valid Loss Unweighted=2.8161 | Stale=0 | ETA=896.3m +rank0 [2026-11-14 04:35:34,699] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:35:35,148] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:35:35,158] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:35,216] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:35:35,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:35:35,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:35:36,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:35:36,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:35:36,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:35:36,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:35:37,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:35:37,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:35:37,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:35:37,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:35:38,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:35:38,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:35:38,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:35:39,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:35:39,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:35:39,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:35:39,544] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:35:39,554] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:40,112] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:40,520] INFO : Plotted number of elements +rank0 [2026-11-14 04:35:41,828] INFO : Plotted jets +rank0 [2026-11-14 04:35:42,790] INFO : Plotted jet ratio +rank0 [2026-11-14 04:36:09,407] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:36:09,408] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:11,868] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:37:11,868] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:35,136] INFO : Step 21000/500000 rank0 | Train Loss: 4.9505 | Train Loss Unweighted: 3.5542 | Train Loss Components: Classification: 0.0069 | Classification_binary: 3.3217 | Regression_cos_phi: 0.0029 | Regression_energy: 0.1104 | Regression_eta: 0.0058 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 3.3217 | Regression_cos_phi: 0.0259 | Regression_energy: 0.6189 | Regression_eta: 0.0347 | Regression_pt: 0.8290 | Regression_sin_phi: 0.0327 | LR: 4.28e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.5840s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1796 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2682.2 | uss_MB: 1527.9 | child_rss_MB: 22629.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:38:14,786] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:38:14,786] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:17,128] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:39:17,128] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:26,672] INFO : Step 22000/500000 rank0 | Train Loss: 4.0479 | Train Loss Unweighted: 2.0364 | Train Loss Components: Classification: 0.0043 | Classification_binary: 1.6998 | Regression_cos_phi: 0.0036 | Regression_energy: 0.1563 | Regression_eta: 0.0067 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 1.6998 | Regression_cos_phi: 0.0330 | Regression_energy: 0.8762 | Regression_eta: 0.0402 | Regression_pt: 1.2606 | Regression_sin_phi: 0.0834 | LR: 4.50e-05 | DataLoad Time: 0.0021s | Model Forward Time: 0.5951s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.2146 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2682.8 | uss_MB: 1528.5 | child_rss_MB: 22628.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:39:26,674] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:39:26,756] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:40:20,194] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:40:20,194] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:18,421] INFO : Step 23000/500000 rank0 | Train Loss: 4.8737 | Train Loss Unweighted: 2.5366 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.1634 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1655 | Regression_eta: 0.0068 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.3096 | Classification_binary: 2.1634 | Regression_cos_phi: 0.0533 | Regression_energy: 0.9274 | Regression_eta: 0.0407 | Regression_pt: 1.3374 | Regression_sin_phi: 0.0420 | LR: 4.73e-05 | DataLoad Time: 0.0028s | Model Forward Time: 0.5645s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0401 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1794 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2684.8 | uss_MB: 1530.5 | child_rss_MB: 22628.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:41:23,081] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:41:23,081] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:25,950] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:42:25,950] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:10,033] INFO : Step 24000/500000 rank0 | Train Loss: 4.2370 | Train Loss Unweighted: 2.1928 | Train Loss Components: Classification: 0.0054 | Classification_binary: 1.8493 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1649 | Regression_eta: 0.0037 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 1.8493 | Regression_cos_phi: 0.0460 | Regression_energy: 0.9244 | Regression_eta: 0.0220 | Regression_pt: 1.2977 | Regression_sin_phi: 0.0285 | LR: 4.97e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5778s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0390 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1948 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2684.8 | uss_MB: 1530.5 | child_rss_MB: 22626.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:43:10,036] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:43:10,116] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:43:28,363] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:43:28,363] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:30,731] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:44:30,731] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:01,762] INFO : Step 25000/500000 rank0 | Train Loss: 3.5155 | Train Loss Unweighted: 2.4016 | Train Loss Components: Classification: 0.0042 | Classification_binary: 2.2168 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0862 | Regression_eta: 0.0029 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 2.2168 | Regression_cos_phi: 0.0285 | Regression_energy: 0.4832 | Regression_eta: 0.0171 | Regression_pt: 0.6781 | Regression_sin_phi: 0.0382 | LR: 5.22e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.5537s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0358 | max_loss: 0.0029 | max_backward: 0.0405 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1707 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2682.2 | uss_MB: 1527.9 | child_rss_MB: 22620.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:45:34,085] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:45:34,085] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:36,614] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:46:36,614] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:53,448] INFO : Step 26000/500000 rank0 | Train Loss: 4.8859 | Train Loss Unweighted: 2.8214 | Train Loss Components: Classification: 0.0034 | Classification_binary: 2.4853 | Regression_cos_phi: 0.0225 | Regression_energy: 0.1348 | Regression_eta: 0.0146 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 2.4853 | Regression_cos_phi: 0.2034 | Regression_energy: 0.7557 | Regression_eta: 0.0874 | Regression_pt: 1.1921 | Regression_sin_phi: 0.1186 | LR: 5.47e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.5879s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0359 | max_loss: 0.0028 | max_backward: 0.0392 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.2104 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2683.0 | uss_MB: 1528.7 | child_rss_MB: 22609.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:46:53,450] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:46:53,532] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:47:39,277] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:47:39,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:41,372] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:48:41,372] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:44,524] INFO : Step 27000/500000 rank0 | Train Loss: 3.8627 | Train Loss Unweighted: 2.0752 | Train Loss Components: Classification: 0.0076 | Classification_binary: 1.7813 | Regression_cos_phi: 0.0048 | Regression_energy: 0.1311 | Regression_eta: 0.0086 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0968 | Classification_binary: 1.7813 | Regression_cos_phi: 0.0431 | Regression_energy: 0.7347 | Regression_eta: 0.0513 | Regression_pt: 1.0551 | Regression_sin_phi: 0.1003 | LR: 5.74e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.6031s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0365 | max_loss: 0.0029 | max_backward: 0.0398 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.2191 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2685.0 | uss_MB: 1530.7 | child_rss_MB: 22629.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:49:44,922] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:49:44,922] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:36,910] INFO : Step 28000/500000 rank0 | Train Loss: 2.1607 | Train Loss Unweighted: 1.4832 | Train Loss Components: Classification: 0.0016 | Classification_binary: 1.3716 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0459 | Regression_eta: 0.0067 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0208 | Classification_binary: 1.3716 | Regression_cos_phi: 0.0350 | Regression_energy: 0.2572 | Regression_eta: 0.0402 | Regression_pt: 0.3967 | Regression_sin_phi: 0.0392 | LR: 6.01e-05 | DataLoad Time: 0.0019s | Model Forward Time: 0.5854s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.2000 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2685.2 | uss_MB: 1530.9 | child_rss_MB: 22625.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:50:36,912] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:50:36,988] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:50:48,158] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:50:48,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:50,750] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:51:50,750] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:28,779] INFO : Step 29000/500000 rank0 | Train Loss: 5.0643 | Train Loss Unweighted: 2.4774 | Train Loss Components: Classification: 0.0051 | Classification_binary: 2.0513 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1803 | Regression_eta: 0.0140 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0153 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 2.0513 | Regression_cos_phi: 0.0824 | Regression_energy: 1.0107 | Regression_eta: 0.0835 | Regression_pt: 1.6273 | Regression_sin_phi: 0.1439 | LR: 6.29e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.6029s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0835 | max_tensorboard_logging: 0.2079 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2685.5 | uss_MB: 1531.2 | child_rss_MB: 22619.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:52:53,745] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:52:53,745] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:56,070] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:53:56,076] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:20,773] INFO : Step 30000/500000 rank0 | Train Loss: 4.4491 | Train Loss Unweighted: 2.6262 | Train Loss Components: Classification: 0.0067 | Classification_binary: 2.3300 | Regression_cos_phi: 0.0036 | Regression_energy: 0.1187 | Regression_eta: 0.0063 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0850 | Classification_binary: 2.3300 | Regression_cos_phi: 0.0324 | Regression_energy: 0.6651 | Regression_eta: 0.0376 | Regression_pt: 1.2731 | Regression_sin_phi: 0.0259 | LR: 6.58e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.5909s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.2011 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2682.8 | uss_MB: 1528.5 | child_rss_MB: 22629.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:54:20,775] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:54:20,849] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:54:20,851] INFO : Running validation on rank0 +rank0 [2026-11-14 04:54:20,851] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:54:20,851] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:54:20,976] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:54:21,082] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:54:21,186] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:54:21,255] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:54:21,324] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:54:21,391] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:54:21,458] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:54:21,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:54:21,637] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:54:21,744] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:54:21,850] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:54:21,918] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:54:21,987] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:54:22,054] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:54:22,121] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:54:22,187] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:54:22,287] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:54:22,292] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:54:22,294] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:54:22,296] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:54:22,298] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:54:22,300] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:54:22,303] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:54:22,304] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:54:22,306] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:54:22,308] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:54:22,310] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:54:22,312] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:54:22,315] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:54:22,317] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:54:22,319] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:54:22,321] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:54:23,767] INFO : VALIDATION | Step=30000/500000 | Train Loss=4.4491 | Valid Loss=6.0118 | Train Loss Unweighted=2.6262 | Valid Loss Unweighted=3.1073 | Stale=1 | ETA=879.9m +rank0 [2026-11-14 04:54:23,768] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:54:24,284] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:54:24,296] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:24,347] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:54:24,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:54:25,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:54:25,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:54:25,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:54:25,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:54:26,090] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:54:26,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:54:26,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:54:26,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:54:27,103] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:54:27,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:54:27,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:54:27,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:54:28,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:54:28,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:54:28,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:54:28,683] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:54:28,691] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:29,263] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:29,679] INFO : Plotted number of elements +rank0 [2026-11-14 04:54:30,966] INFO : Plotted jets +rank0 [2026-11-14 04:54:31,921] INFO : Plotted jet ratio +rank0 [2026-11-14 04:55:10,899] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:55:10,899] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:13,582] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:56:13,582] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:24,173] INFO : Step 31000/500000 rank0 | Train Loss: 2.8018 | Train Loss Unweighted: 1.4986 | Train Loss Components: Classification: 0.0032 | Classification_binary: 1.2814 | Regression_cos_phi: 0.0096 | Regression_energy: 0.0932 | Regression_eta: 0.0148 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 1.2814 | Regression_cos_phi: 0.0869 | Regression_energy: 0.5226 | Regression_eta: 0.0886 | Regression_pt: 0.7368 | Regression_sin_phi: 0.0446 | LR: 6.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.6123s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0366 | max_loss: 0.0031 | max_backward: 0.0402 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.2201 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2796.7 | uss_MB: 1642.5 | child_rss_MB: 22629.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:57:16,351] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:57:16,352] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:15,279] INFO : Step 32000/500000 rank0 | Train Loss: 3.6138 | Train Loss Unweighted: 1.7641 | Train Loss Components: Classification: 0.0049 | Classification_binary: 1.4598 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1333 | Regression_eta: 0.0227 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0365 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 1.4598 | Regression_cos_phi: 0.0505 | Regression_energy: 0.7472 | Regression_eta: 0.1357 | Regression_pt: 0.8147 | Regression_sin_phi: 0.3430 | LR: 7.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5460s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1531 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2796.7 | uss_MB: 1642.5 | child_rss_MB: 22627.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:58:15,281] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:58:15,363] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:58:19,202] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:58:19,202] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:21,739] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:59:21,739] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:06,009] INFO : Step 33000/500000 rank0 | Train Loss: 2.7133 | Train Loss Unweighted: 1.6880 | Train Loss Components: Classification: 0.0112 | Classification_binary: 1.5263 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0677 | Regression_eta: 0.0054 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1427 | Classification_binary: 1.5263 | Regression_cos_phi: 0.0433 | Regression_energy: 0.3797 | Regression_eta: 0.0321 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0335 | LR: 7.51e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5557s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0386 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1711 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2707.4 | uss_MB: 1553.1 | child_rss_MB: 22625.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:00:23,532] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:00:23,532] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:26,944] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:01:26,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:00,347] INFO : Step 34000/500000 rank0 | Train Loss: 3.6205 | Train Loss Unweighted: 1.8775 | Train Loss Components: Classification: 0.0061 | Classification_binary: 1.5864 | Regression_cos_phi: 0.0032 | Regression_energy: 0.1347 | Regression_eta: 0.0071 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 1.5864 | Regression_cos_phi: 0.0290 | Regression_energy: 0.7548 | Regression_eta: 0.0421 | Regression_pt: 1.1065 | Regression_sin_phi: 0.0232 | LR: 7.83e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.6301s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0031 | max_backward: 0.0417 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.2231 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2707.9 | uss_MB: 1553.6 | child_rss_MB: 22610.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:02:00,350] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:02:00,431] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:02:32,565] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:02:32,566] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:34,951] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:03:34,951] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:53,198] INFO : Step 35000/500000 rank0 | Train Loss: 2.8437 | Train Loss Unweighted: 2.0443 | Train Loss Components: Classification: 0.0083 | Classification_binary: 1.9195 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0493 | Regression_eta: 0.0046 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1061 | Classification_binary: 1.9195 | Regression_cos_phi: 0.0428 | Regression_energy: 0.2761 | Regression_eta: 0.0276 | Regression_pt: 0.4323 | Regression_sin_phi: 0.0394 | LR: 8.16e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.5785s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1899 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2709.9 | uss_MB: 1555.6 | child_rss_MB: 22630.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:04:38,050] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:04:38,050] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:40,357] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:05:40,358] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:44,318] INFO : Step 36000/500000 rank0 | Train Loss: 2.2937 | Train Loss Unweighted: 1.1014 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.9001 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0921 | Regression_eta: 0.0073 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0181 | Classification_binary: 0.9001 | Regression_cos_phi: 0.0183 | Regression_energy: 0.5160 | Regression_eta: 0.0433 | Regression_pt: 0.7622 | Regression_sin_phi: 0.0358 | LR: 8.50e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.5436s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0368 | max_loss: 0.0031 | max_backward: 0.0404 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1448 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2709.9 | uss_MB: 1555.7 | child_rss_MB: 22628.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 05:05:44,320] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:05:44,397] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:06:42,837] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:06:42,837] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:35,441] INFO : Step 37000/500000 rank0 | Train Loss: 1.5534 | Train Loss Unweighted: 0.6330 | Train Loss Components: Classification: 0.0015 | Classification_binary: 0.4819 | Regression_cos_phi: 0.0033 | Regression_energy: 0.0577 | Regression_eta: 0.0134 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0196 | Classification_binary: 0.4819 | Regression_cos_phi: 0.0302 | Regression_energy: 0.3233 | Regression_eta: 0.0800 | Regression_pt: 0.5208 | Regression_sin_phi: 0.0975 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5582s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0358 | max_loss: 0.0029 | max_backward: 0.0401 | max_optimizer_step: 0.0779 | max_tensorboard_logging: 0.1666 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2710.0 | uss_MB: 1555.7 | child_rss_MB: 22627.7 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:07:45,515] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:07:45,515] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:47,845] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:08:47,845] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:27,033] INFO : Step 38000/500000 rank0 | Train Loss: 2.5373 | Train Loss Unweighted: 1.0192 | Train Loss Components: Classification: 0.0022 | Classification_binary: 0.7626 | Regression_cos_phi: 0.0035 | Regression_energy: 0.1217 | Regression_eta: 0.0047 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.7626 | Regression_cos_phi: 0.0312 | Regression_energy: 0.6824 | Regression_eta: 0.0281 | Regression_pt: 0.9812 | Regression_sin_phi: 0.0234 | LR: 9.21e-05 | DataLoad Time: 0.0022s | Model Forward Time: 0.5785s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0404 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1866 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2707.9 | uss_MB: 1553.7 | child_rss_MB: 22622.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:09:27,035] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:09:27,116] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:09:51,383] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:09:51,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:53,817] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:10:53,817] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:18,938] INFO : Step 39000/500000 rank0 | Train Loss: 2.6553 | Train Loss Unweighted: 1.3425 | Train Loss Components: Classification: 0.0207 | Classification_binary: 1.1429 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0814 | Regression_eta: 0.0047 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.2641 | Classification_binary: 1.1429 | Regression_cos_phi: 0.0625 | Regression_energy: 0.4561 | Regression_eta: 0.0280 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0777 | LR: 9.57e-05 | DataLoad Time: 0.0023s | Model Forward Time: 0.5742s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0385 | max_loss: 0.0032 | max_backward: 0.0395 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1917 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2710.0 | uss_MB: 1555.8 | child_rss_MB: 22629.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:11:56,849] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:11:56,849] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:59,914] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:12:59,915] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:11,398] INFO : Step 40000/500000 rank0 | Train Loss: 1.5230 | Train Loss Unweighted: 0.6471 | Train Loss Components: Classification: 0.0025 | Classification_binary: 0.5007 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0667 | Regression_eta: 0.0075 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0322 | Classification_binary: 0.5007 | Regression_cos_phi: 0.0260 | Regression_energy: 0.3740 | Regression_eta: 0.0446 | Regression_pt: 0.4911 | Regression_sin_phi: 0.0544 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5708s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0408 | max_loss: 0.0033 | max_backward: 0.0420 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1836 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2710.4 | uss_MB: 1556.1 | child_rss_MB: 22629.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:13:11,401] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:13:11,474] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:13:11,476] INFO : Running validation on rank0 +rank0 [2026-11-14 05:13:11,476] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:13:11,476] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:13:11,604] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:13:11,710] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:13:11,810] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:13:11,879] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:13:11,945] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:13:12,011] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:13:12,080] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:13:12,147] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:13:12,261] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:13:12,371] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:13:12,481] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:13:12,550] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:13:12,620] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:13:12,687] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:13:12,755] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:13:12,823] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:13:12,934] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:13:12,940] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:13:12,943] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:13:12,944] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:13:12,947] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:13:12,949] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:13:12,951] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:13:12,952] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:13:12,954] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:13:12,956] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:13:12,959] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:13:12,961] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:13:12,963] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:13:12,964] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:13:12,966] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:13:12,968] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:13:14,433] INFO : VALIDATION | Step=40000/500000 | Train Loss=1.5230 | Valid Loss=8.1474 | Train Loss Unweighted=0.6471 | Valid Loss Unweighted=4.6317 | Stale=2 | ETA=862.6m +rank0 [2026-11-14 05:13:14,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:13:15,012] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:13:15,027] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:15,076] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:13:15,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:13:15,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:13:16,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:13:16,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:13:16,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:13:16,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:13:17,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:13:17,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:13:17,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:13:17,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:13:18,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:13:18,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:13:18,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:13:18,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:13:19,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:13:19,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:13:19,369] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:13:19,377] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:19,924] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:20,341] INFO : Plotted number of elements +rank0 [2026-11-14 05:13:21,631] INFO : Plotted jets +rank0 [2026-11-14 05:13:22,577] INFO : Plotted jet ratio +rank0 [2026-11-14 05:14:14,663] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:14:14,663] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:14,438] INFO : Step 41000/500000 rank0 | Train Loss: 2.9571 | Train Loss Unweighted: 1.3747 | Train Loss Components: Classification: 0.0039 | Classification_binary: 1.1072 | Regression_cos_phi: 0.0018 | Regression_energy: 0.1289 | Regression_eta: 0.0048 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 1.1072 | Regression_cos_phi: 0.0164 | Regression_energy: 0.7226 | Regression_eta: 0.0287 | Regression_pt: 1.0150 | Regression_sin_phi: 0.0172 | LR: 1.03e-04 | DataLoad Time: 0.0015s | Model Forward Time: 0.5375s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0398 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1528 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2733.2 | uss_MB: 1578.9 | child_rss_MB: 22627.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:15:17,304] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:15:17,305] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:19,484] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:16:19,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:05,598] INFO : Step 42000/500000 rank0 | Train Loss: 1.2687 | Train Loss Unweighted: 0.5117 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.3862 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0535 | Regression_eta: 0.0052 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0070 | Classification_binary: 0.3862 | Regression_cos_phi: 0.0378 | Regression_energy: 0.3001 | Regression_eta: 0.0313 | Regression_pt: 0.4564 | Regression_sin_phi: 0.0500 | LR: 1.07e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.5586s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0453 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1762 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2733.6 | uss_MB: 1579.4 | child_rss_MB: 22616.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:17:05,600] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:17:05,680] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:17:22,366] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:17:22,366] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:25,088] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:18:25,088] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:57,475] INFO : Step 43000/500000 rank0 | Train Loss: 1.7196 | Train Loss Unweighted: 0.7058 | Train Loss Components: Classification: 0.0075 | Classification_binary: 0.5398 | Regression_cos_phi: 0.0065 | Regression_energy: 0.0746 | Regression_eta: 0.0115 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0953 | Classification_binary: 0.5398 | Regression_cos_phi: 0.0592 | Regression_energy: 0.4184 | Regression_eta: 0.0689 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0656 | LR: 1.11e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5873s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0031 | max_backward: 0.0401 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.2016 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2735.4 | uss_MB: 1581.2 | child_rss_MB: 22613.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:19:27,976] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:19:27,977] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:30,266] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:20:30,266] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:48,709] INFO : Step 44000/500000 rank0 | Train Loss: 1.3215 | Train Loss Unweighted: 0.5040 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.3690 | Regression_cos_phi: 0.0036 | Regression_energy: 0.0540 | Regression_eta: 0.0089 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0033 | Classification_binary: 0.3690 | Regression_cos_phi: 0.0327 | Regression_energy: 0.3026 | Regression_eta: 0.0533 | Regression_pt: 0.4780 | Regression_sin_phi: 0.0826 | LR: 1.15e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5258s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1470 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2736.4 | uss_MB: 1582.1 | child_rss_MB: 22630.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:20:48,711] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:20:48,785] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:21:33,011] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:21:33,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:35,460] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:22:35,461] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:40,690] INFO : Step 45000/500000 rank0 | Train Loss: 1.0263 | Train Loss Unweighted: 0.4690 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.3752 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0412 | Regression_eta: 0.0051 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0012 | Classification_binary: 0.3752 | Regression_cos_phi: 0.0274 | Regression_energy: 0.2310 | Regression_eta: 0.0304 | Regression_pt: 0.3295 | Regression_sin_phi: 0.0315 | LR: 1.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.5603s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1760 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2736.4 | uss_MB: 1582.2 | child_rss_MB: 22629.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:23:39,110] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:23:39,110] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:34,700] INFO : Step 46000/500000 rank0 | Train Loss: 0.8002 | Train Loss Unweighted: 0.2205 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.1227 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0429 | Regression_eta: 0.0060 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0027 | Classification_binary: 0.1227 | Regression_cos_phi: 0.0256 | Regression_energy: 0.2403 | Regression_eta: 0.0359 | Regression_pt: 0.3469 | Regression_sin_phi: 0.0259 | LR: 1.23e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.5604s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0395 | max_optimizer_step: 0.0823 | max_tensorboard_logging: 0.1622 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2734.0 | uss_MB: 1579.8 | child_rss_MB: 22629.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:24:34,702] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 05:24:34,780] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:24:44,109] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 05:24:44,109] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:45,844] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 05:25:45,844] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:26,032] INFO : Step 47000/500000 rank0 | Train Loss: 1.1431 | Train Loss Unweighted: 0.4855 | Train Loss Components: Classification: 0.0094 | Classification_binary: 0.3848 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0372 | Regression_eta: 0.0068 | Regression_pt: 0.0398 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.1198 | Classification_binary: 0.3848 | Regression_cos_phi: 0.0276 | Regression_energy: 0.2086 | Regression_eta: 0.0406 | Regression_pt: 0.3201 | Regression_sin_phi: 0.0414 | LR: 1.27e-04 | DataLoad Time: 0.0023s | Model Forward Time: 0.6016s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.2078 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2735.8 | uss_MB: 1581.6 | child_rss_MB: 22624.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:26:49,321] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 05:26:49,321] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:52,350] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 05:27:52,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:18,478] INFO : Step 48000/500000 rank0 | Train Loss: 0.9457 | Train Loss Unweighted: 0.2672 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0161 | Regression_energy: 0.0366 | Regression_eta: 0.0048 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0232 | Classification_binary: 0.1610 | Regression_cos_phi: 0.1452 | Regression_energy: 0.2052 | Regression_eta: 0.0288 | Regression_pt: 0.3446 | Regression_sin_phi: 0.0376 | LR: 1.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.6345s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.2486 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2735.8 | uss_MB: 1581.6 | child_rss_MB: 22619.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:28:18,480] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 05:28:18,555] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:28:55,525] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:28:55,525] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:29:57,707] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:29:57,707] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:09,930] INFO : Step 49000/500000 rank0 | Train Loss: 1.9961 | Train Loss Unweighted: 1.4607 | Train Loss Components: Classification: 0.0006 | Classification_binary: 1.3725 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0357 | Regression_eta: 0.0069 | Regression_pt: 0.0344 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0080 | Classification_binary: 1.3725 | Regression_cos_phi: 0.0448 | Regression_energy: 0.2002 | Regression_eta: 0.0411 | Regression_pt: 0.2767 | Regression_sin_phi: 0.0529 | LR: 1.36e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5311s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0386 | max_loss: 0.0030 | max_backward: 0.0390 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1490 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2735.8 | uss_MB: 1581.6 | child_rss_MB: 22627.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:31:00,378] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:31:00,378] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:01,583] INFO : Step 50000/500000 rank0 | Train Loss: 1.2646 | Train Loss Unweighted: 0.7548 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.6693 | Regression_cos_phi: 0.0015 | Regression_energy: 0.0388 | Regression_eta: 0.0017 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 12.7508 | Classification_binary: 1.0000 | Regression_cos_phi: 9.0456 | Regression_energy: 5.6049 | Regression_eta: 5.9681 | Regression_pt: 8.0442 | Regression_sin_phi: 9.4010 | Task Weighted Losses: Classification: 0.0048 | Classification_binary: 0.6693 | Regression_cos_phi: 0.0137 | Regression_energy: 0.2175 | Regression_eta: 0.0102 | Regression_pt: 0.3340 | Regression_sin_phi: 0.0151 | LR: 1.40e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.5900s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0391 | max_optimizer_step: 0.0871 | max_tensorboard_logging: 0.1869 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2733.4 | uss_MB: 1579.1 | child_rss_MB: 22622.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:32:01,585] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:32:01,668] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:32:01,671] INFO : Running validation on rank0 +rank0 [2026-11-14 05:32:01,671] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:32:01,671] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:32:01,795] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:32:01,898] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:32:01,998] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:32:02,068] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:32:02,134] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:32:02,199] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:32:02,265] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:32:02,332] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:32:02,443] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:32:02,550] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:32:02,655] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:32:02,723] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:32:02,789] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:32:02,857] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:32:02,925] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:32:02,994] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:32:03,105] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:32:03,110] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:32:03,112] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:32:03,115] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:32:03,117] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:32:03,120] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:32:03,122] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:32:03,124] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:32:03,126] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:32:03,129] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:32:03,131] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:32:03,133] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:32:03,136] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:32:03,137] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:32:03,139] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:32:03,141] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:32:04,589] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.2646 | Valid Loss=10.2431 | Train Loss Unweighted=0.7548 | Valid Loss Unweighted=6.7029 | Stale=3 | ETA=844.6m +rank0 [2026-11-14 05:32:04,589] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:32:05,057] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:32:05,072] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:05,123] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:32:05,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:32:05,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:32:06,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:32:06,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:32:06,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:32:06,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:32:07,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:32:07,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:32:07,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:32:07,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:32:08,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:32:08,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:32:08,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:32:08,837] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:32:09,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:32:09,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:32:09,368] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:32:09,375] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:09,915] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_1p5mm_seed12345_20261114_035700_604077/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:10,315] INFO : Plotted number of elements +rank0 [2026-11-14 05:32:11,582] INFO : Plotted jets +rank0 [2026-11-14 05:32:12,505] INFO : Plotted jet ratio +rank0 [2026-11-14 05:32:12,911] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:32:12,911] INFO : Training completed. Total time on device 0: 93.985min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..b5637fc9bfe030f099320f177ab461a93e3b4ea2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfaed50f9b6be6ac251234f6cdd2a882fc11844ea8b7669b72f606ee00864105 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-50000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..b71872757ed5a5fe50262acf58d114221d18d0d6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b24ee0ab293094fca8f9b7823ea7aab6e00c328dc45e68cc8a6bcbf01b6e921 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..8c3a6db9d2071cce7d5120a6023f9248b3761081 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 2.0719246864318848, "Classification": 0.006858853157609701, "Regression_pt": 0.08618886768817902, "Regression_eta": 0.012449911795556545, "Regression_sin_phi": 0.09924516081809998, "Regression_cos_phi": 0.11105702072381973, "Regression_energy": 0.07378891110420227, "Total": 5.275087356567383, "Total_unweighted": 2.461513411719352}, "train_group": {}, "valid": {"Classification_binary": 2.686521530151367, "Classification": 0.021385284140706062, "Regression_pt": 0.21083134412765503, "Regression_eta": 0.01582890935242176, "Regression_sin_phi": 0.01167185977101326, "Regression_cos_phi": 0.018491923809051514, "Regression_energy": 0.21606501936912537, "Total": 6.309292793273926, "Total_unweighted": 3.18079587072134, "metrics/particle/class_1/efficiency": 0.8522953987121582, "metrics/particle/class_1/pid_accuracy": 0.9414519667625427, "metrics/particle/class_2/efficiency": 0.09090909361839294, "metrics/particle/class_2/pid_accuracy": 0.0, "metrics/particle/class_3/efficiency": 0.554973840713501, "metrics/particle/class_3/pid_accuracy": 0.9591194987297058, "metrics/particle/class_4/efficiency": 0.821052610874176, "metrics/particle/class_4/pid_accuracy": 0.8589743375778198, "metrics/particle/class_5/efficiency": 0.9277108311653137, "metrics/particle/class_5/pid_accuracy": 0.9610389471054077, "metrics/particle/count/bias_mean": 0.00800000037997961, "metrics/particle/count/mae": 0.671999990940094, "metrics/particle/count/prediction_mean": 5.104000091552734, "metrics/particle/count/target_mean": 5.0960001945495605, "metrics/particle/event/energy_relative_abs_error": 0.09184730798006058, "metrics/particle/event/energy_response_mean": 1.010958194732666, "metrics/particle/event/met_abs_error": 3.0697665214538574, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09020112454891205, "metrics/particle/event/scalar_pt_response_mean": 1.0160568952560425, "metrics/particle/event/vector_pt_closure": 0.10612638294696808, "metrics/particle/matched/delta_eta_abs_mean": 0.02715766243636608, "metrics/particle/matched/delta_phi_abs_mean": 0.021657753735780716, "metrics/particle/matched/delta_r_mean": 0.03856511414051056, "metrics/particle/matched/energy_relative_abs_mean": 0.09160294383764267, "metrics/particle/matched/pid_accuracy": 0.9401330351829529, "metrics/particle/matched/pt_relative_abs_mean": 0.0909905806183815, "metrics/particle/matching/duplicate_fraction": 0.03369905799627304, "metrics/particle/matching/efficiency": 0.7080062627792358, "metrics/particle/matching/f1": 0.7074509859085083, "metrics/particle/matching/purity": 0.7068965435028076, "metrics/particle/matching/target_coverage_dr0p05": 0.5180533528327942, "metrics/particle/matching/target_coverage_dr0p10": 0.7401883602142334, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6758241653442383, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8163265585899353}, "step_train_time": 0.5353190898895264, "step_valid_time": 2.7521878289990127, "step_total_time": 1119.2234790325165, "elapsed_training_loop_seconds": 1119.2234790325165, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0085043472962556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09354433034229781, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 388, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 382, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9086651053864169, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9086651053864169, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9086651053864169, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09133489461358314, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8946135831381733, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8946135831381733, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8946135831381733, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9086651053864169, "step_test_time": 4.913209314050619, "step_plot_time": 3.7223417469649576, "step_validation_cycle_time": 11.89633384597255} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..6933ca5fe6fe6273f9fa570b262178e36fca1d3e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 1.4348115921020508, "Classification": 0.008471202105283737, "Regression_pt": 0.09407742321491241, "Regression_eta": 0.004091836977750063, "Regression_sin_phi": 0.003910078667104244, "Regression_cos_phi": 0.0037213547620922327, "Regression_energy": 0.08778736740350723, "Total": 2.9174692630767822, "Total_unweighted": 1.6368708552327007}, "train_group": {}, "valid": {"Classification_binary": 2.4926018714904785, "Classification": 0.017619574442505836, "Regression_pt": 0.19637814164161682, "Regression_eta": 0.011677371338009834, "Regression_sin_phi": 0.010613786056637764, "Regression_cos_phi": 0.016363093629479408, "Regression_energy": 0.20268724858760834, "Total": 5.816726207733154, "Total_unweighted": 2.9479410871863365, "metrics/particle/class_1/efficiency": 0.910179615020752, "metrics/particle/class_1/pid_accuracy": 0.9473684430122375, "metrics/particle/class_2/efficiency": 0.13636364042758942, "metrics/particle/class_2/pid_accuracy": 0.3333333432674408, "metrics/particle/class_3/efficiency": 0.5811518430709839, "metrics/particle/class_3/pid_accuracy": 0.9759759902954102, "metrics/particle/class_4/efficiency": 0.821052610874176, "metrics/particle/class_4/pid_accuracy": 0.9230769276618958, "metrics/particle/class_5/efficiency": 0.9879518151283264, "metrics/particle/class_5/pid_accuracy": 0.9878048896789551, "metrics/particle/count/bias_mean": -0.2919999957084656, "metrics/particle/count/mae": 0.6039999723434448, "metrics/particle/count/prediction_mean": 4.803999900817871, "metrics/particle/count/target_mean": 5.0960001945495605, "metrics/particle/event/energy_relative_abs_error": 0.08109983801841736, "metrics/particle/event/energy_response_mean": 0.9847778081893921, "metrics/particle/event/met_abs_error": 2.538388967514038, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08028209209442139, "metrics/particle/event/scalar_pt_response_mean": 0.9856570363044739, "metrics/particle/event/vector_pt_closure": 0.09397654980421066, "metrics/particle/matched/delta_eta_abs_mean": 0.024612655863165855, "metrics/particle/matched/delta_phi_abs_mean": 0.019493846222758293, "metrics/particle/matched/delta_r_mean": 0.03485165163874626, "metrics/particle/matched/energy_relative_abs_mean": 0.08925310522317886, "metrics/particle/matched/pid_accuracy": 0.956932783126831, "metrics/particle/matched/pt_relative_abs_mean": 0.08827477693557739, "metrics/particle/matching/duplicate_fraction": 0.01665278896689415, "metrics/particle/matching/efficiency": 0.7472527623176575, "metrics/particle/matching/f1": 0.769292950630188, "metrics/particle/matching/purity": 0.7926727533340454, "metrics/particle/matching/target_coverage_dr0p05": 0.5855572819709778, "metrics/particle/matching/target_coverage_dr0p10": 0.7715855836868286, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6899529099464417, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8304552435874939}, "step_train_time": 0.5214474201202393, "step_valid_time": 2.9077981989830732, "step_total_time": 2246.449330329895, "elapsed_training_loop_seconds": 2246.449330329895, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0092964747531605, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07823858814918239, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 401, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 395, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9391100702576113, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9413145539906104, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.94021101992966, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05868544600938967, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9250585480093677, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9272300469483568, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9261430246189918, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9391100702576113, "step_test_time": 4.646595601981971, "step_plot_time": 3.176696587994229, "step_validation_cycle_time": 11.25192904699361} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..d82b4947141349c95633b203273cac3e28c44059 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 1.8763298988342285, "Classification": 0.009554348886013031, "Regression_pt": 0.07301955670118332, "Regression_eta": 0.006397808436304331, "Regression_sin_phi": 0.004290834069252014, "Regression_cos_phi": 0.00591751653701067, "Regression_energy": 0.07593051344156265, "Total": 3.1678123474121094, "Total_unweighted": 2.0514404769055545}, "train_group": {}, "valid": {"Classification_binary": 2.7286434173583984, "Classification": 0.019377749413251877, "Regression_pt": 0.23874136805534363, "Regression_eta": 0.01192166842520237, "Regression_sin_phi": 0.01135250274091959, "Regression_cos_phi": 0.013505646958947182, "Regression_energy": 0.24252334237098694, "Total": 6.6356329917907715, "Total_unweighted": 3.26606569532305, "metrics/particle/class_1/efficiency": 0.916167676448822, "metrics/particle/class_1/pid_accuracy": 0.943355143070221, "metrics/particle/class_2/efficiency": 0.13636364042758942, "metrics/particle/class_2/pid_accuracy": 0.3333333432674408, "metrics/particle/class_3/efficiency": 0.5881326198577881, "metrics/particle/class_3/pid_accuracy": 0.9584569931030273, "metrics/particle/class_4/efficiency": 0.8631578683853149, "metrics/particle/class_4/pid_accuracy": 0.9268292784690857, "metrics/particle/class_5/efficiency": 0.9879518151283264, "metrics/particle/class_5/pid_accuracy": 0.9878048896789551, "metrics/particle/count/bias_mean": -0.06400000303983688, "metrics/particle/count/mae": 0.5680000185966492, "metrics/particle/count/prediction_mean": 5.0320000648498535, "metrics/particle/count/target_mean": 5.0960001945495605, "metrics/particle/event/energy_relative_abs_error": 0.08190050721168518, "metrics/particle/event/energy_response_mean": 1.0022300481796265, "metrics/particle/event/met_abs_error": 2.7011783123016357, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08281683176755905, "metrics/particle/event/scalar_pt_response_mean": 1.00493586063385, "metrics/particle/event/vector_pt_closure": 0.10037313401699066, "metrics/particle/matched/delta_eta_abs_mean": 0.024778185412287712, "metrics/particle/matched/delta_phi_abs_mean": 0.020547199994325638, "metrics/particle/matched/delta_r_mean": 0.03552239015698433, "metrics/particle/matched/energy_relative_abs_mean": 0.08953256905078888, "metrics/particle/matched/pid_accuracy": 0.9491173624992371, "metrics/particle/matched/pt_relative_abs_mean": 0.08815541118383408, "metrics/particle/matching/duplicate_fraction": 0.023052463307976723, "metrics/particle/matching/efficiency": 0.755886971950531, "metrics/particle/matching/f1": 0.7606635093688965, "metrics/particle/matching/purity": 0.7655007839202881, "metrics/particle/matching/target_coverage_dr0p05": 0.5894819498062134, "metrics/particle/matching/target_coverage_dr0p10": 0.7896389365196228, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7025117874145508, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8383045792579651}, "step_train_time": 0.5173015594482422, "step_valid_time": 2.8862563319853507, "step_total_time": 3372.7198433876038, "elapsed_training_loop_seconds": 3372.7198433876038, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994616878226401, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0830584844332295, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 426, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 402, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 393, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9414519906323185, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9436619718309859, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9425556858147714, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056338028169014086, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9203747072599532, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9225352112676056, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9214536928487691, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9414519906323185, "step_test_time": 4.654789781954605, "step_plot_time": 3.1255021169781685, "step_validation_cycle_time": 11.116008513025008} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..73ad53233945afc46994530ddfea3bfec0488cdb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 0.25035592913627625, "Classification": 0.00026504829293116927, "Regression_pt": 0.06056802719831467, "Regression_eta": 0.0202044527977705, "Regression_sin_phi": 0.003965836483985186, "Regression_cos_phi": 0.19168363511562347, "Regression_energy": 0.05783575773239136, "Total": 3.0005176067352295, "Total_unweighted": 0.5848786867572926}, "train_group": {}, "valid": {"Classification_binary": 4.693091869354248, "Classification": 0.03539912775158882, "Regression_pt": 0.25628405809402466, "Regression_eta": 0.012033405713737011, "Regression_sin_phi": 0.011573029682040215, "Regression_cos_phi": 0.011829989962279797, "Regression_energy": 0.256602942943573, "Total": 9.015144348144531, "Total_unweighted": 5.2768144235014915, "metrics/particle/class_1/efficiency": 0.9041916131973267, "metrics/particle/class_1/pid_accuracy": 0.940397322177887, "metrics/particle/class_2/efficiency": 0.13636364042758942, "metrics/particle/class_2/pid_accuracy": 0.6666666865348816, "metrics/particle/class_3/efficiency": 0.6055846214294434, "metrics/particle/class_3/pid_accuracy": 0.9596541523933411, "metrics/particle/class_4/efficiency": 0.8526315689086914, "metrics/particle/class_4/pid_accuracy": 0.9135802388191223, "metrics/particle/class_5/efficiency": 0.9759036302566528, "metrics/particle/class_5/pid_accuracy": 0.9629629850387573, "metrics/particle/count/bias_mean": 0.15600000321865082, "metrics/particle/count/mae": 0.6759999990463257, "metrics/particle/count/prediction_mean": 5.251999855041504, "metrics/particle/count/target_mean": 5.0960001945495605, "metrics/particle/event/energy_relative_abs_error": 0.10026486217975616, "metrics/particle/event/energy_response_mean": 1.0268350839614868, "metrics/particle/event/met_abs_error": 2.752887487411499, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09881633520126343, "metrics/particle/event/scalar_pt_response_mean": 1.0309008359909058, "metrics/particle/event/vector_pt_closure": 0.10692144930362701, "metrics/particle/matched/delta_eta_abs_mean": 0.02540541999042034, "metrics/particle/matched/delta_phi_abs_mean": 0.020604919642210007, "metrics/particle/matched/delta_r_mean": 0.03628775477409363, "metrics/particle/matched/energy_relative_abs_mean": 0.09215104579925537, "metrics/particle/matched/pid_accuracy": 0.9461140036582947, "metrics/particle/matched/pt_relative_abs_mean": 0.09306861460208893, "metrics/particle/matching/duplicate_fraction": 0.02817974053323269, "metrics/particle/matching/efficiency": 0.7574568390846252, "metrics/particle/matching/f1": 0.7460379004478455, "metrics/particle/matching/purity": 0.7349581122398376, "metrics/particle/matching/target_coverage_dr0p05": 0.5879120826721191, "metrics/particle/matching/target_coverage_dr0p10": 0.7864992022514343, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7001569867134094, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8437990546226501}, "step_train_time": 0.5229721069335938, "step_valid_time": 2.9613347810227424, "step_total_time": 4503.0636875629425, "elapsed_training_loop_seconds": 4503.0636875629425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0290366538076436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0731983895760362, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 398, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9461358313817331, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9395348837209302, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9428238039673279, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06046511627906977, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9320843091334895, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9255813953488372, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9288214702450408, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9461358313817331, "step_test_time": 4.683143892965745, "step_plot_time": 3.194976851984393, "step_validation_cycle_time": 11.323585562990047} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..85a49a69bc7b965b0bd2a8674aead39443f6939a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 0.5680248737335205, "Classification": 0.0014090090990066528, "Regression_pt": 0.06491699814796448, "Regression_eta": 0.005144355818629265, "Regression_sin_phi": 0.0039162891916930676, "Regression_cos_phi": 0.0026869382709264755, "Regression_energy": 0.06929463893175125, "Total": 1.6110718250274658, "Total_unweighted": 0.7153931031934917}, "train_group": {}, "valid": {"Classification_binary": 5.547128677368164, "Classification": 0.048895105719566345, "Regression_pt": 0.27639055252075195, "Regression_eta": 0.012601262889802456, "Regression_sin_phi": 0.0099935969337821, "Regression_cos_phi": 0.01193607784807682, "Regression_energy": 0.28006547689437866, "Total": 10.329926490783691, "Total_unweighted": 6.187010750174522, "metrics/particle/class_1/efficiency": 0.910179615020752, "metrics/particle/class_1/pid_accuracy": 0.9517543911933899, "metrics/particle/class_2/efficiency": 0.09090909361839294, "metrics/particle/class_2/pid_accuracy": 0.5, "metrics/particle/class_3/efficiency": 0.5916230082511902, "metrics/particle/class_3/pid_accuracy": 0.9587020874023438, "metrics/particle/class_4/efficiency": 0.8526315689086914, "metrics/particle/class_4/pid_accuracy": 0.9259259104728699, "metrics/particle/class_5/efficiency": 0.9879518151283264, "metrics/particle/class_5/pid_accuracy": 0.9390243887901306, "metrics/particle/count/bias_mean": 0.08399999886751175, "metrics/particle/count/mae": 0.628000020980835, "metrics/particle/count/prediction_mean": 5.179999828338623, "metrics/particle/count/target_mean": 5.0960001945495605, "metrics/particle/event/energy_relative_abs_error": 0.08712522685527802, "metrics/particle/event/energy_response_mean": 0.9965773224830627, "metrics/particle/event/met_abs_error": 2.4294087886810303, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08539421856403351, "metrics/particle/event/scalar_pt_response_mean": 0.9943791627883911, "metrics/particle/event/vector_pt_closure": 0.09651248157024384, "metrics/particle/matched/delta_eta_abs_mean": 0.024091660976409912, "metrics/particle/matched/delta_phi_abs_mean": 0.020453354343771935, "metrics/particle/matched/delta_r_mean": 0.034953150898218155, "metrics/particle/matched/energy_relative_abs_mean": 0.07815088331699371, "metrics/particle/matched/pid_accuracy": 0.949999988079071, "metrics/particle/matched/pt_relative_abs_mean": 0.07644808292388916, "metrics/particle/matching/duplicate_fraction": 0.016988417133688927, "metrics/particle/matching/efficiency": 0.7535321712493896, "metrics/particle/matching/f1": 0.7473725080490112, "metrics/particle/matching/purity": 0.7413127422332764, "metrics/particle/matching/target_coverage_dr0p05": 0.5894819498062134, "metrics/particle/matching/target_coverage_dr0p10": 0.779434859752655, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7307692170143127, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8414442539215088}, "step_train_time": 0.5262422561645508, "step_valid_time": 2.910034954024013, "step_total_time": 5638.497114419937, "elapsed_training_loop_seconds": 5638.497114419937, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9928266340230274, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.052418352968584436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 425, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 402, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 393, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9414519906323185, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9458823529411765, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9436619718309859, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05411764705882353, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9203747072599532, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9247058823529412, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9225352112676056, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9414519906323185, "step_test_time": 4.672498514992185, "step_plot_time": 3.165850337012671, "step_validation_cycle_time": 11.18819552497007} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..52182c7cf4cf062685a4b0692bca61fd840d32b7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_z_tautau_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..27e538f01c4084738a8d809f290fd8c223880ad0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b6bf49fa3864c3ae2d0894d25d45db76119450071bebdc352cfb24301592b93 +size 4402 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..bb325b334345f1ab52f85cb18976665b3a8e9a32 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..13ef0ddfa9f6be58ab9c4138983870c39c0b1855 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f3da0350fd1cd6253c3e071f28ca41fd226e0f43 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..e93a4c89f7c1aad6d0c4a7fb733a1189ce5961cf Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9507a1bab61c137c35873c75d6acb4572ea57c27 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..e7ada938b3346a20aa5defe2c87316828380e1ae Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..602993d379ab52edc66ea4715f66bdaa833be216 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..492e44f1f0f1781f585c0952de7d73a22dcbaca2 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..38d787fb2609020a963ed61396b854cad9b4814d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..8411bf857daf033c2e8e1ca7b665ccbd7d39735f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b821c30f51461a2ee01b4ae30c1132beb2cdf8f6 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..3e5dc3f358bcd35a219c2e655be7459c540ec774 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/runs/train/events.out.tfevents.1794646706.workergpu161.1995819.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/runs/train/events.out.tfevents.1794646706.workergpu161.1995819.0 new file mode 100644 index 0000000000000000000000000000000000000000..d655c9485b8c0ab2fab8379d2d24d9135330d9be --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/runs/train/events.out.tfevents.1794646706.workergpu161.1995819.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a11d2f3b61346a6e55f9a07878a74a8b57c1a513ea24d210881d6a20b0d5d26 +size 1843423 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/runs/valid/events.out.tfevents.1794646706.workergpu161.1995819.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/runs/valid/events.out.tfevents.1794646706.workergpu161.1995819.1 new file mode 100644 index 0000000000000000000000000000000000000000..2fa94d305773a3a99d544f951730689beaadb008 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/runs/valid/events.out.tfevents.1794646706.workergpu161.1995819.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ed5b35617be3daed8149d37981d763a662c8b07356a3cf5650205e916a3458a +size 2239365 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..67ac1de31866cca7f2dbfd8042d0042656b0b7fc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_2mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..121f78b63bcea0a6d484eb3a76060bc87cb868b2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6855cc0e41d9d8ed4445c9c10d2274c9b2b9b909 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/train.log @@ -0,0 +1,999 @@ +rank0 [2026-11-14 03:58:22,987] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-14 03:58:22,987] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 03:58:23,969] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 03:58:23,969] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 03:58:23,970] INFO : Instantiating model +rank0 [2026-11-14 03:58:24,000] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 03:58:24,000] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 03:58:24,002] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 03:58:24,002] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 03:58:24,002] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 03:58:24,003] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 03:58:24,003] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,005] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 03:58:24,005] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,008] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 03:58:24,008] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,011] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 03:58:24,011] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,014] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 03:58:24,014] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,017] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 03:58:24,017] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,019] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 03:58:24,020] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 03:58:24,020] INFO : Initializing output DNNs +rank0 [2026-11-14 03:58:24,028] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,031] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,038] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-14 03:58:24,038] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 03:58:24,038] INFO : jagged_attention enabled=True +rank0 [2026-11-14 03:58:24,038] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 03:58:24,038] INFO : task_queries enabled=True +rank0 [2026-11-14 03:58:24,038] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 03:58:24,038] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 03:58:24,038] INFO : classification_query parameters: 256 +rank0 [2026-11-14 03:58:24,038] INFO : regression_query parameters: 256 +rank0 [2026-11-14 03:58:24,039] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 03:58:24,039] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 03:58:24,039] INFO : output_mode=elementwise +rank0 [2026-11-14 03:58:24,039] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 03:58:24,039] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 03:58:24,039] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 03:58:24,039] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 03:58:24,039] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 03:58:24,039] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 03:58:24,039] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 03:58:24,039] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 03:58:24,040] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 03:58:24,040] INFO : MLPF __init__ done +rank0 [2026-11-14 03:58:24,040] INFO : Instantiated model +rank0 [2026-11-14 03:58:24,043] INFO : Moving model to device rank=0 +rank0 [2026-11-14 03:58:24,054] INFO : Moved model to device rank=0 +rank0 [2026-11-14 03:58:24,055] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 03:58:24,158] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 03:58:24,158] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 03:58:24,159] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 03:58:24,159] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 03:58:24,159] INFO : Total parameters: 3910930 +rank0 [2026-11-14 03:58:24,162] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 03:58:24,164] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597 +rank0 [2026-11-14 03:58:24,164] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597 +rank0 [2026-11-14 03:58:24,164] INFO : Getting dataloaders +rank0 [2026-11-14 03:58:24,164] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:58:26,028] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,070] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:26,115] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:26,148] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,174] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:26,204] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,230] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,262] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,304] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:26,419] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,419] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 03:58:26,419] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 03:58:26,419] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 03:58:26,420] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:58:26,427] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,434] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,441] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,448] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,455] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,463] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,470] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,478] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,485] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,492] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,492] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 03:58:26,492] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 03:58:26,547] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:58:26,684] INFO : Got dataloaders +rank0 [2026-11-14 03:58:26,684] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 03:58:26,684] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 03:58:26,684] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 03:58:26,690] INFO : Creating train_iterator +rank0 [2026-11-14 03:58:26,690] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 03:58:26,715] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:58:26,860] INFO : Created train_iterator +rank0 [2026-11-14 03:58:39,547] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.46730902671814, 'Classification': 0.5930971106886864, 'Regression_pt': 0.8945067819952964, 'Regression_eta': 1.1884842801094055, 'Regression_sin_phi': 0.8354161743819714, 'Regression_cos_phi': 0.8169610878825188, 'Regression_energy': 1.3217136013507842}, weights={'Classification_binary': 1.0, 'Classification': 12.590364456176758, 'Regression_pt': 8.347962379455566, 'Regression_eta': 6.283052444458008, 'Regression_sin_phi': 8.938429832458496, 'Regression_cos_phi': 9.140348434448242, 'Regression_energy': 5.649717807769775} +rank0 [2026-11-14 03:59:31,269] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 03:59:31,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:00:20,218] INFO : Step 1000/500000 rank0 | Train Loss: 11.3437 | Train Loss Unweighted: 5.1166 | Train Loss Components: Classification: 0.1328 | Classification_binary: 4.2277 | Regression_cos_phi: 0.0391 | Regression_energy: 0.3094 | Regression_eta: 0.0478 | Regression_pt: 0.3012 | Regression_sin_phi: 0.0586 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 1.6726 | Classification_binary: 4.2277 | Regression_cos_phi: 0.3575 | Regression_energy: 1.7478 | Regression_eta: 0.3005 | Regression_pt: 2.5144 | Regression_sin_phi: 0.5234 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5028s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0032 | max_backward: 0.0410 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1332 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2566.3 | uss_MB: 1397.9 | child_rss_MB: 22698.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:00:33,944] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:00:33,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:36,334] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:01:36,334] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:02:11,716] INFO : Step 2000/500000 rank0 | Train Loss: 8.7946 | Train Loss Unweighted: 4.4529 | Train Loss Components: Classification: 0.0430 | Classification_binary: 3.7877 | Regression_cos_phi: 0.0229 | Regression_energy: 0.2483 | Regression_eta: 0.0441 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0235 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.5419 | Classification_binary: 3.7877 | Regression_cos_phi: 0.2094 | Regression_energy: 1.4027 | Regression_eta: 0.2772 | Regression_pt: 2.3655 | Regression_sin_phi: 0.2104 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5131s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0379 | max_loss: 0.0032 | max_backward: 0.0400 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.1266 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2567.1 | uss_MB: 1398.7 | child_rss_MB: 22627.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:02:11,719] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:02:39,392] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:02:39,392] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:41,799] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:03:41,800] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:03,365] INFO : Step 3000/500000 rank0 | Train Loss: 6.0888 | Train Loss Unweighted: 3.2458 | Train Loss Components: Classification: 0.0324 | Classification_binary: 2.8118 | Regression_cos_phi: 0.0236 | Regression_energy: 0.1741 | Regression_eta: 0.0218 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0217 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.4074 | Classification_binary: 2.8118 | Regression_cos_phi: 0.2157 | Regression_energy: 0.9834 | Regression_eta: 0.1371 | Regression_pt: 1.3397 | Regression_sin_phi: 0.1937 | LR: 2.05e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4809s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0343 | max_loss: 0.0026 | max_backward: 0.0404 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1198 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.2 | uss_MB: 1402.2 | child_rss_MB: 22628.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:04:44,761] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:04:44,762] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:47,051] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:05:47,051] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:54,755] INFO : Step 4000/500000 rank0 | Train Loss: 8.1598 | Train Loss Unweighted: 3.2610 | Train Loss Components: Classification: 0.0341 | Classification_binary: 2.4828 | Regression_cos_phi: 0.0075 | Regression_energy: 0.3527 | Regression_eta: 0.0115 | Regression_pt: 0.3640 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.4293 | Classification_binary: 2.4828 | Regression_cos_phi: 0.0685 | Regression_energy: 1.9927 | Regression_eta: 0.0720 | Regression_pt: 3.0383 | Regression_sin_phi: 0.0762 | LR: 2.08e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4972s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0031 | max_backward: 0.0405 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1230 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.2 | uss_MB: 1402.3 | child_rss_MB: 22627.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:05:54,757] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:06:49,674] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:06:49,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:45,989] INFO : Step 5000/500000 rank0 | Train Loss: 5.2044 | Train Loss Unweighted: 3.1028 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.7782 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1327 | Regression_eta: 0.0138 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.2820 | Classification_binary: 2.7782 | Regression_cos_phi: 0.0500 | Regression_energy: 0.7497 | Regression_eta: 0.0870 | Regression_pt: 1.1945 | Regression_sin_phi: 0.0630 | LR: 2.13e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1194 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2599.0 | uss_MB: 1431.1 | child_rss_MB: 22622.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:07:52,381] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:07:52,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:54,354] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:08:54,354] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:36,869] INFO : Step 6000/500000 rank0 | Train Loss: 6.5604 | Train Loss Unweighted: 3.3120 | Train Loss Components: Classification: 0.0257 | Classification_binary: 2.8114 | Regression_cos_phi: 0.0525 | Regression_energy: 0.1764 | Regression_eta: 0.0554 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0164 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.3234 | Classification_binary: 2.8114 | Regression_cos_phi: 0.4801 | Regression_energy: 0.9968 | Regression_eta: 0.3479 | Regression_pt: 1.4545 | Regression_sin_phi: 0.1463 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5239s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0386 | max_loss: 0.0030 | max_backward: 0.0411 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1215 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2599.0 | uss_MB: 1431.1 | child_rss_MB: 22620.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:09:36,871] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:09:57,142] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:09:57,142] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:59,777] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:10:59,777] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:27,518] INFO : Step 7000/500000 rank0 | Train Loss: 7.4770 | Train Loss Unweighted: 3.3970 | Train Loss Components: Classification: 0.0110 | Classification_binary: 2.7294 | Regression_cos_phi: 0.0063 | Regression_energy: 0.3194 | Regression_eta: 0.0095 | Regression_pt: 0.3144 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1384 | Classification_binary: 2.7294 | Regression_cos_phi: 0.0574 | Regression_energy: 1.8047 | Regression_eta: 0.0597 | Regression_pt: 2.6243 | Regression_sin_phi: 0.0631 | LR: 2.26e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4591s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0026 | max_backward: 0.0376 | max_optimizer_step: 0.0589 | max_tensorboard_logging: 0.1272 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2599.1 | uss_MB: 1431.2 | child_rss_MB: 22620.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:12:01,863] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:12:01,863] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:05,518] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:13:05,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:20,171] INFO : Step 8000/500000 rank0 | Train Loss: 5.2526 | Train Loss Unweighted: 2.8218 | Train Loss Components: Classification: 0.0304 | Classification_binary: 2.4563 | Regression_cos_phi: 0.0130 | Regression_energy: 0.1423 | Regression_eta: 0.0063 | Regression_pt: 0.1681 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.3822 | Classification_binary: 2.4563 | Regression_cos_phi: 0.1184 | Regression_energy: 0.8040 | Regression_eta: 0.0395 | Regression_pt: 1.4034 | Regression_sin_phi: 0.0487 | LR: 2.34e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4568s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0356 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0597 | max_tensorboard_logging: 0.1197 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2599.2 | uss_MB: 1431.3 | child_rss_MB: 22612.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 20.8 +rank0 [2026-11-14 04:13:20,172] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:13:20,240] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:14:06,597] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:14:06,597] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:09,690] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:15:09,690] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:10,978] INFO : Step 9000/500000 rank0 | Train Loss: 5.0189 | Train Loss Unweighted: 3.1561 | Train Loss Components: Classification: 0.0103 | Classification_binary: 2.8590 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1326 | Regression_eta: 0.0095 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1298 | Classification_binary: 2.8590 | Regression_cos_phi: 0.0953 | Regression_energy: 0.7490 | Regression_eta: 0.0599 | Regression_pt: 1.0491 | Regression_sin_phi: 0.0769 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5086s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0026 | max_backward: 0.0406 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.1366 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2567.8 | uss_MB: 1400.0 | child_rss_MB: 22596.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:16:12,538] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:16:12,538] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:17:03,070] INFO : Step 10000/500000 rank0 | Train Loss: 5.2751 | Train Loss Unweighted: 2.4615 | Train Loss Components: Classification: 0.0069 | Classification_binary: 2.0719 | Regression_cos_phi: 0.1111 | Regression_energy: 0.0738 | Regression_eta: 0.0124 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0992 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0864 | Classification_binary: 2.0719 | Regression_cos_phi: 1.0151 | Regression_energy: 0.4169 | Regression_eta: 0.0782 | Regression_pt: 0.7195 | Regression_sin_phi: 0.8871 | LR: 2.52e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5344s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0031 | max_backward: 0.0412 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1416 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2568.5 | uss_MB: 1400.5 | child_rss_MB: 22617.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:17:03,072] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:17:03,154] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:17:03,154] INFO : Running validation on rank0 +rank0 [2026-11-14 04:17:03,154] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:17:03,154] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:17:03,354] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:17:03,564] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:17:03,634] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:17:03,690] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:17:03,746] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:17:03,796] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:17:03,852] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:17:03,910] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:17:03,959] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:17:04,029] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:17:04,100] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:17:04,156] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:17:04,214] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:17:04,264] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:17:04,322] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:17:04,380] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:17:04,431] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:17:04,436] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:17:04,438] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:17:04,440] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:17:04,441] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:17:04,444] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:17:04,445] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:17:04,447] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:17:04,449] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:17:04,451] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:17:04,453] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:17:04,455] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:17:04,455] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:17:04,456] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:17:04,459] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:17:04,462] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:17:05,985] INFO : VALIDATION | Step=10000/500000 | Train Loss=5.2751 | Valid Loss=6.3093 | Train Loss Unweighted=2.4615 | Valid Loss Unweighted=3.1808 | Stale=0 | ETA=914.1m +rank0 [2026-11-14 04:17:05,986] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:17:06,348] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:17:06,356] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:06,404] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:17:07,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:17:07,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:17:07,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:17:07,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:17:08,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:17:08,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:17:08,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:17:08,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:17:09,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:17:09,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:17:09,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:17:09,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:17:10,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:17:10,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:17:10,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:17:10,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:17:10,897] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:17:10,904] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:11,445] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:12,337] INFO : Plotted number of elements +rank0 [2026-11-14 04:17:13,673] INFO : Plotted jets +rank0 [2026-11-14 04:17:14,613] INFO : Plotted jet ratio +rank0 [2026-11-14 04:17:27,899] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:17:27,899] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:30,148] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:18:30,148] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:06,357] INFO : Step 11000/500000 rank0 | Train Loss: 6.0337 | Train Loss Unweighted: 3.5951 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.2019 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1824 | Regression_eta: 0.0111 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1672 | Classification_binary: 3.2019 | Regression_cos_phi: 0.0573 | Regression_energy: 1.0304 | Regression_eta: 0.0697 | Regression_pt: 1.4595 | Regression_sin_phi: 0.0477 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5149s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0412 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2700.4 | uss_MB: 1545.7 | child_rss_MB: 22627.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:19:33,022] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:19:33,022] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:35,509] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:20:35,509] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:57,908] INFO : Step 12000/500000 rank0 | Train Loss: 3.2011 | Train Loss Unweighted: 1.6724 | Train Loss Components: Classification: 0.0254 | Classification_binary: 1.4470 | Regression_cos_phi: 0.0064 | Regression_energy: 0.0805 | Regression_eta: 0.0128 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.3194 | Classification_binary: 1.4470 | Regression_cos_phi: 0.0583 | Regression_energy: 0.4547 | Regression_eta: 0.0806 | Regression_pt: 0.7802 | Regression_sin_phi: 0.0608 | LR: 2.75e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4982s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0398 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1192 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2700.4 | uss_MB: 1545.7 | child_rss_MB: 22629.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:20:57,910] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:20:57,982] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:21:38,435] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:21:38,435] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:41,333] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:22:41,334] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:49,980] INFO : Step 13000/500000 rank0 | Train Loss: 5.6114 | Train Loss Unweighted: 2.7311 | Train Loss Components: Classification: 0.0139 | Classification_binary: 2.2691 | Regression_cos_phi: 0.0035 | Regression_energy: 0.2057 | Regression_eta: 0.0122 | Regression_pt: 0.2198 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1750 | Classification_binary: 2.2691 | Regression_cos_phi: 0.0323 | Regression_energy: 1.1624 | Regression_eta: 0.0767 | Regression_pt: 1.8345 | Regression_sin_phi: 0.0614 | LR: 2.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5342s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0032 | max_backward: 0.0415 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1258 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2653.7 | uss_MB: 1499.0 | child_rss_MB: 22625.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:23:44,172] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:23:44,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:41,324] INFO : Step 14000/500000 rank0 | Train Loss: 7.0186 | Train Loss Unweighted: 3.4251 | Train Loss Components: Classification: 0.0060 | Classification_binary: 2.8562 | Regression_cos_phi: 0.0680 | Regression_energy: 0.2337 | Regression_eta: 0.0202 | Regression_pt: 0.2306 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 2.8562 | Regression_cos_phi: 0.6219 | Regression_energy: 1.3204 | Regression_eta: 0.1270 | Regression_pt: 1.9253 | Regression_sin_phi: 0.0922 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5108s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0399 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1267 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2657.9 | uss_MB: 1503.1 | child_rss_MB: 22625.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:41,326] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:24:41,400] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:24:46,944] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:24:46,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:49,210] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:25:49,210] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:32,920] INFO : Step 15000/500000 rank0 | Train Loss: 5.9356 | Train Loss Unweighted: 2.6748 | Train Loss Components: Classification: 0.0145 | Classification_binary: 2.1700 | Regression_cos_phi: 0.0353 | Regression_energy: 0.2035 | Regression_eta: 0.0073 | Regression_pt: 0.2002 | Regression_sin_phi: 0.0439 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1830 | Classification_binary: 2.1700 | Regression_cos_phi: 0.3229 | Regression_energy: 1.1496 | Regression_eta: 0.0460 | Regression_pt: 1.6716 | Regression_sin_phi: 0.3924 | LR: 3.17e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4928s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1302 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2660.5 | uss_MB: 1505.8 | child_rss_MB: 22623.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:26:52,226] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:26:52,226] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:54,328] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:27:54,329] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:23,866] INFO : Step 16000/500000 rank0 | Train Loss: 5.8052 | Train Loss Unweighted: 2.7445 | Train Loss Components: Classification: 0.0102 | Classification_binary: 2.2459 | Regression_cos_phi: 0.0067 | Regression_energy: 0.2360 | Regression_eta: 0.0101 | Regression_pt: 0.2248 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1287 | Classification_binary: 2.2459 | Regression_cos_phi: 0.0617 | Regression_energy: 1.3333 | Regression_eta: 0.0633 | Regression_pt: 1.8762 | Regression_sin_phi: 0.0961 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5077s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0356 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0710 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2660.6 | uss_MB: 1505.9 | child_rss_MB: 22618.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:28:23,868] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:23,937] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:28:56,703] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:28:56,703] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:59,314] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:29:59,314] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:15,158] INFO : Step 17000/500000 rank0 | Train Loss: 8.2892 | Train Loss Unweighted: 2.6973 | Train Loss Components: Classification: 0.0141 | Classification_binary: 1.7826 | Regression_cos_phi: 0.0049 | Regression_energy: 0.4385 | Regression_eta: 0.0059 | Regression_pt: 0.4478 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1769 | Classification_binary: 1.7826 | Regression_cos_phi: 0.0446 | Regression_energy: 2.4772 | Regression_eta: 0.0372 | Regression_pt: 3.7383 | Regression_sin_phi: 0.0323 | LR: 3.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4965s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0026 | max_backward: 0.0393 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1238 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2657.0 | uss_MB: 1502.3 | child_rss_MB: 22604.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:31:02,130] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:31:02,130] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:04,596] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:32:04,596] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:06,773] INFO : Step 18000/500000 rank0 | Train Loss: 7.0760 | Train Loss Unweighted: 2.9533 | Train Loss Components: Classification: 0.0081 | Classification_binary: 2.2810 | Regression_cos_phi: 0.0107 | Regression_energy: 0.3111 | Regression_eta: 0.0110 | Regression_pt: 0.3276 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1021 | Classification_binary: 2.2810 | Regression_cos_phi: 0.0981 | Regression_energy: 1.7579 | Regression_eta: 0.0693 | Regression_pt: 2.7346 | Regression_sin_phi: 0.0330 | LR: 3.69e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4820s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0029 | max_backward: 0.0413 | max_optimizer_step: 0.0638 | max_tensorboard_logging: 0.1193 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2657.7 | uss_MB: 1503.0 | child_rss_MB: 22630.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:32:06,775] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:32:06,844] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:33:07,124] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:33:07,125] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:57,652] INFO : Step 19000/500000 rank0 | Train Loss: 5.0555 | Train Loss Unweighted: 2.4955 | Train Loss Components: Classification: 0.0098 | Classification_binary: 2.0867 | Regression_cos_phi: 0.0126 | Regression_energy: 0.1742 | Regression_eta: 0.0150 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1232 | Classification_binary: 2.0867 | Regression_cos_phi: 0.1151 | Regression_energy: 0.9841 | Regression_eta: 0.0945 | Regression_pt: 1.5717 | Regression_sin_phi: 0.0802 | LR: 3.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5119s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0381 | max_loss: 0.0031 | max_backward: 0.0416 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1247 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2660.4 | uss_MB: 1505.7 | child_rss_MB: 22624.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:34:09,634] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:34:09,634] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:12,367] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:35:12,367] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:50,160] INFO : Step 20000/500000 rank0 | Train Loss: 2.9175 | Train Loss Unweighted: 1.6369 | Train Loss Components: Classification: 0.0085 | Classification_binary: 1.4348 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0878 | Regression_eta: 0.0041 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1067 | Classification_binary: 1.4348 | Regression_cos_phi: 0.0340 | Regression_energy: 0.4960 | Regression_eta: 0.0257 | Regression_pt: 0.7854 | Regression_sin_phi: 0.0349 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0033 | max_backward: 0.0404 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1322 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2660.8 | uss_MB: 1506.0 | child_rss_MB: 22630.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:50,162] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:35:50,229] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:35:50,230] INFO : Running validation on rank0 +rank0 [2026-11-14 04:35:50,230] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:35:50,230] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:35:50,352] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:35:50,462] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:35:50,566] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:35:50,636] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:35:50,704] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:35:50,771] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:35:50,841] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:35:50,911] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:35:51,012] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:35:51,117] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:35:51,219] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:35:51,287] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:35:51,355] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:35:51,420] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:35:51,489] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:35:51,558] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:35:51,659] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:35:51,665] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:35:51,667] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:35:51,669] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:35:51,671] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:35:51,673] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:35:51,675] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:35:51,677] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:35:51,679] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:35:51,681] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:35:51,683] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:35:51,685] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:35:51,686] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:35:51,687] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:35:51,689] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:35:51,692] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:35:53,209] INFO : VALIDATION | Step=20000/500000 | Train Loss=2.9175 | Valid Loss=5.8167 | Train Loss Unweighted=1.6369 | Valid Loss Unweighted=2.9479 | Stale=0 | ETA=898.6m +rank0 [2026-11-14 04:35:53,210] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:35:53,611] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:35:53,619] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:53,659] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:35:54,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:35:54,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:35:54,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:35:54,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:35:55,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:35:55,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:35:55,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:35:55,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:35:56,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:35:56,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:35:56,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:35:56,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:35:57,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:35:57,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:35:57,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:35:57,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:35:57,854] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:35:57,861] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:58,398] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:58,769] INFO : Plotted number of elements +rank0 [2026-11-14 04:36:00,060] INFO : Plotted jets +rank0 [2026-11-14 04:36:01,026] INFO : Plotted jet ratio +rank0 [2026-11-14 04:36:27,410] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:36:27,410] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:30,216] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:37:30,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:53,486] INFO : Step 21000/500000 rank0 | Train Loss: 3.1722 | Train Loss Unweighted: 1.8618 | Train Loss Components: Classification: 0.0017 | Classification_binary: 1.6532 | Regression_cos_phi: 0.0100 | Regression_energy: 0.0866 | Regression_eta: 0.0047 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0214 | Classification_binary: 1.6532 | Regression_cos_phi: 0.0915 | Regression_energy: 0.4893 | Regression_eta: 0.0295 | Regression_pt: 0.7967 | Regression_sin_phi: 0.0906 | LR: 4.28e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5348s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0029 | max_backward: 0.0396 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1420 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2691.4 | uss_MB: 1536.7 | child_rss_MB: 22633.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:38:32,977] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:38:32,977] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:34,349] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:39:34,349] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:43,476] INFO : Step 22000/500000 rank0 | Train Loss: 3.7398 | Train Loss Unweighted: 1.6836 | Train Loss Components: Classification: 0.0134 | Classification_binary: 1.3556 | Regression_cos_phi: 0.0030 | Regression_energy: 0.1496 | Regression_eta: 0.0056 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1690 | Classification_binary: 1.3556 | Regression_cos_phi: 0.0270 | Regression_energy: 0.8452 | Regression_eta: 0.0351 | Regression_pt: 1.2755 | Regression_sin_phi: 0.0323 | LR: 4.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4722s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0026 | max_backward: 0.0380 | max_optimizer_step: 0.0596 | max_tensorboard_logging: 0.1254 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2692.4 | uss_MB: 1537.8 | child_rss_MB: 22634.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:39:43,478] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:39:43,548] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:40:38,525] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:40:38,525] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:36,511] INFO : Step 23000/500000 rank0 | Train Loss: 3.4357 | Train Loss Unweighted: 2.2239 | Train Loss Components: Classification: 0.0094 | Classification_binary: 2.0360 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0754 | Regression_eta: 0.0081 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1189 | Classification_binary: 2.0360 | Regression_cos_phi: 0.0664 | Regression_energy: 0.4263 | Regression_eta: 0.0511 | Regression_pt: 0.6543 | Regression_sin_phi: 0.0827 | LR: 4.73e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0028 | max_backward: 0.0389 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1292 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2695.9 | uss_MB: 1541.2 | child_rss_MB: 22636.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:41:41,009] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:41:41,009] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:43,005] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:42:43,005] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:27,853] INFO : Step 24000/500000 rank0 | Train Loss: 5.0597 | Train Loss Unweighted: 2.2145 | Train Loss Components: Classification: 0.0036 | Classification_binary: 1.7285 | Regression_cos_phi: 0.0136 | Regression_energy: 0.1123 | Regression_eta: 0.2218 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0154 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 1.7285 | Regression_cos_phi: 0.1239 | Regression_energy: 0.6342 | Regression_eta: 1.3936 | Regression_pt: 0.9960 | Regression_sin_phi: 0.1377 | LR: 4.97e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5374s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1492 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2695.9 | uss_MB: 1541.2 | child_rss_MB: 22631.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:43:27,855] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:43:27,927] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:43:46,437] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:43:46,437] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:49,305] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:44:49,305] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:20,116] INFO : Step 25000/500000 rank0 | Train Loss: 2.9067 | Train Loss Unweighted: 1.3899 | Train Loss Components: Classification: 0.0039 | Classification_binary: 1.1419 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1153 | Regression_eta: 0.0072 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 1.1419 | Regression_cos_phi: 0.0306 | Regression_energy: 0.6516 | Regression_eta: 0.0453 | Regression_pt: 0.9646 | Regression_sin_phi: 0.0231 | LR: 5.22e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5109s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0027 | max_backward: 0.0390 | max_optimizer_step: 0.0666 | max_tensorboard_logging: 0.1327 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2691.5 | uss_MB: 1536.8 | child_rss_MB: 22620.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:45:52,240] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:45:52,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:54,576] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:46:54,576] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:11,370] INFO : Step 26000/500000 rank0 | Train Loss: 2.4612 | Train Loss Unweighted: 1.0077 | Train Loss Components: Classification: 0.0017 | Classification_binary: 0.7699 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1117 | Regression_eta: 0.0028 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0212 | Classification_binary: 0.7699 | Regression_cos_phi: 0.0559 | Regression_energy: 0.6310 | Regression_eta: 0.0177 | Regression_pt: 0.9442 | Regression_sin_phi: 0.0211 | LR: 5.47e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5176s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0027 | max_backward: 0.0405 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1322 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2691.9 | uss_MB: 1537.2 | child_rss_MB: 22628.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:47:11,372] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:47:11,442] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:47:57,172] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:47:57,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:59,107] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:48:59,107] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:02,164] INFO : Step 27000/500000 rank0 | Train Loss: 4.3758 | Train Loss Unweighted: 2.2342 | Train Loss Components: Classification: 0.0129 | Classification_binary: 1.8957 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1475 | Regression_eta: 0.0076 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1620 | Classification_binary: 1.8957 | Regression_cos_phi: 0.0773 | Regression_energy: 0.8332 | Regression_eta: 0.0478 | Regression_pt: 1.2600 | Regression_sin_phi: 0.0998 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5058s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0364 | max_loss: 0.0031 | max_backward: 0.0395 | max_optimizer_step: 0.0646 | max_tensorboard_logging: 0.1267 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2695.4 | uss_MB: 1540.7 | child_rss_MB: 22640.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:50:02,011] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:50:02,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:53,272] INFO : Step 28000/500000 rank0 | Train Loss: 2.3043 | Train Loss Unweighted: 1.4392 | Train Loss Components: Classification: 0.0110 | Classification_binary: 1.3068 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0536 | Regression_eta: 0.0085 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1389 | Classification_binary: 1.3068 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3028 | Regression_eta: 0.0532 | Regression_pt: 0.4015 | Regression_sin_phi: 0.0554 | LR: 6.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5034s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0031 | max_backward: 0.0395 | max_optimizer_step: 0.0661 | max_tensorboard_logging: 0.1346 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2695.6 | uss_MB: 1540.9 | child_rss_MB: 22637.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:50:53,274] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:50:53,343] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:51:04,288] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:51:04,288] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:06,383] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:52:06,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:44,301] INFO : Step 29000/500000 rank0 | Train Loss: 2.5663 | Train Loss Unweighted: 1.8859 | Train Loss Components: Classification: 0.0058 | Classification_binary: 1.7797 | Regression_cos_phi: 0.0024 | Regression_energy: 0.0395 | Regression_eta: 0.0106 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 1.7797 | Regression_cos_phi: 0.0221 | Regression_energy: 0.2230 | Regression_eta: 0.0664 | Regression_pt: 0.3715 | Regression_sin_phi: 0.0307 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4996s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0035 | max_backward: 0.0400 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2696.5 | uss_MB: 1541.9 | child_rss_MB: 22636.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:53:09,264] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:53:09,264] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:12,213] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:54:12,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:36,449] INFO : Step 30000/500000 rank0 | Train Loss: 3.1678 | Train Loss Unweighted: 2.0514 | Train Loss Components: Classification: 0.0096 | Classification_binary: 1.8763 | Regression_cos_phi: 0.0059 | Regression_energy: 0.0759 | Regression_eta: 0.0064 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1203 | Classification_binary: 1.8763 | Regression_cos_phi: 0.0541 | Regression_energy: 0.4290 | Regression_eta: 0.0402 | Regression_pt: 0.6096 | Regression_sin_phi: 0.0384 | LR: 6.58e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5166s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0388 | max_loss: 0.0031 | max_backward: 0.0405 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1238 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2692.2 | uss_MB: 1537.5 | child_rss_MB: 22640.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:54:36,451] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:54:36,521] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:54:36,521] INFO : Running validation on rank0 +rank0 [2026-11-14 04:54:36,521] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:54:36,521] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:54:36,644] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:54:36,752] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:54:36,856] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:54:36,925] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:54:36,991] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:54:37,056] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:54:37,122] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:54:37,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:54:37,293] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:54:37,394] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:54:37,494] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:54:37,564] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:54:37,632] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:54:37,701] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:54:37,768] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:54:37,837] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:54:37,936] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:54:37,942] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:54:37,945] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:54:37,947] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:54:37,949] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:54:37,951] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:54:37,952] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:54:37,953] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:54:37,955] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:54:37,957] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:54:37,959] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:54:37,961] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:54:37,962] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:54:37,964] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:54:37,964] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:54:37,966] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:54:39,413] INFO : VALIDATION | Step=30000/500000 | Train Loss=3.1678 | Valid Loss=6.6356 | Train Loss Unweighted=2.0514 | Valid Loss Unweighted=3.2661 | Stale=1 | ETA=880.7m +rank0 [2026-11-14 04:54:39,413] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:54:39,798] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:54:39,805] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:39,854] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:54:40,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:54:40,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:54:40,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:54:41,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:54:41,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:54:41,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:54:41,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:54:42,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:54:42,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:54:42,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:54:42,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:54:43,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:54:43,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:54:43,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:54:43,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:54:44,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:54:44,066] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:54:44,073] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:44,613] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:44,967] INFO : Plotted number of elements +rank0 [2026-11-14 04:54:46,251] INFO : Plotted jets +rank0 [2026-11-14 04:54:47,187] INFO : Plotted jet ratio +rank0 [2026-11-14 04:55:26,573] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:55:26,573] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:29,349] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:56:29,349] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:39,768] INFO : Step 31000/500000 rank0 | Train Loss: 1.9812 | Train Loss Unweighted: 1.3334 | Train Loss Components: Classification: 0.0014 | Classification_binary: 1.2307 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0405 | Regression_eta: 0.0057 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0180 | Classification_binary: 1.2307 | Regression_cos_phi: 0.0549 | Regression_energy: 0.2290 | Regression_eta: 0.0358 | Regression_pt: 0.3577 | Regression_sin_phi: 0.0550 | LR: 6.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5170s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0030 | max_backward: 0.0410 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2800.6 | uss_MB: 1645.9 | child_rss_MB: 22641.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:57:32,176] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:57:32,176] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:31,392] INFO : Step 32000/500000 rank0 | Train Loss: 4.0279 | Train Loss Unweighted: 1.9769 | Train Loss Components: Classification: 0.0084 | Classification_binary: 1.6465 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1506 | Regression_eta: 0.0084 | Regression_pt: 0.1448 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1054 | Classification_binary: 1.6465 | Regression_cos_phi: 0.0555 | Regression_energy: 0.8510 | Regression_eta: 0.0531 | Regression_pt: 1.2087 | Regression_sin_phi: 0.1076 | LR: 7.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5358s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0032 | max_backward: 0.0403 | max_optimizer_step: 0.0666 | max_tensorboard_logging: 0.1405 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2800.6 | uss_MB: 1645.9 | child_rss_MB: 22640.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:58:31,394] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:58:31,464] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:58:35,231] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:58:35,231] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:38,137] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:59:38,137] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:23,359] INFO : Step 33000/500000 rank0 | Train Loss: 3.6434 | Train Loss Unweighted: 1.1686 | Train Loss Components: Classification: 0.0012 | Classification_binary: 0.8319 | Regression_cos_phi: 0.1878 | Regression_energy: 0.0560 | Regression_eta: 0.0094 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0295 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0145 | Classification_binary: 0.8319 | Regression_cos_phi: 1.7165 | Regression_energy: 0.3166 | Regression_eta: 0.0590 | Regression_pt: 0.4413 | Regression_sin_phi: 0.2637 | LR: 7.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5344s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0031 | max_backward: 0.0401 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1381 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2716.9 | uss_MB: 1562.2 | child_rss_MB: 22637.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:00:40,837] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:00:40,837] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:43,478] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:01:43,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:15,102] INFO : Step 34000/500000 rank0 | Train Loss: 1.7795 | Train Loss Unweighted: 1.0908 | Train Loss Components: Classification: 0.0029 | Classification_binary: 0.9815 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0454 | Regression_eta: 0.0060 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.9815 | Regression_cos_phi: 0.0609 | Regression_energy: 0.2562 | Regression_eta: 0.0374 | Regression_pt: 0.3631 | Regression_sin_phi: 0.0441 | LR: 7.83e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5078s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2717.6 | uss_MB: 1563.0 | child_rss_MB: 22624.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:02:15,104] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:02:15,174] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:02:46,790] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:02:46,790] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:49,310] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:03:49,310] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:07,424] INFO : Step 35000/500000 rank0 | Train Loss: 3.4663 | Train Loss Unweighted: 1.1965 | Train Loss Components: Classification: 0.0052 | Classification_binary: 0.8256 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1719 | Regression_eta: 0.0114 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.8256 | Regression_cos_phi: 0.0667 | Regression_energy: 0.9710 | Regression_eta: 0.0719 | Regression_pt: 1.4049 | Regression_sin_phi: 0.0605 | LR: 8.16e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5081s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0389 | max_optimizer_step: 0.0629 | max_tensorboard_logging: 0.1403 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2722.1 | uss_MB: 1567.4 | child_rss_MB: 22649.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:04:52,435] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:04:52,435] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:55,124] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:05:55,125] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:59,133] INFO : Step 36000/500000 rank0 | Train Loss: 1.3828 | Train Loss Unweighted: 0.8913 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.8113 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0344 | Regression_eta: 0.0054 | Regression_pt: 0.0332 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0091 | Classification_binary: 0.8113 | Regression_cos_phi: 0.0361 | Regression_energy: 0.1941 | Regression_eta: 0.0338 | Regression_pt: 0.2773 | Regression_sin_phi: 0.0212 | LR: 8.50e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5236s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0031 | max_backward: 0.0413 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1334 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2722.1 | uss_MB: 1567.4 | child_rss_MB: 22647.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:05:59,135] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:05:59,205] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:06:58,352] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:06:58,352] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:49,854] INFO : Step 37000/500000 rank0 | Train Loss: 1.6053 | Train Loss Unweighted: 0.9423 | Train Loss Components: Classification: 0.0054 | Classification_binary: 0.8389 | Regression_cos_phi: 0.0090 | Regression_energy: 0.0343 | Regression_eta: 0.0185 | Regression_pt: 0.0297 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.8389 | Regression_cos_phi: 0.0825 | Regression_energy: 0.1937 | Regression_eta: 0.1164 | Regression_pt: 0.2478 | Regression_sin_phi: 0.0585 | LR: 8.85e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4522s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0337 | max_loss: 0.0026 | max_backward: 0.0378 | max_optimizer_step: 0.0593 | max_tensorboard_logging: 0.1166 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2722.1 | uss_MB: 1567.5 | child_rss_MB: 22647.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:07:59,603] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:07:59,603] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:04,453] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:09:04,453] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:42,551] INFO : Step 38000/500000 rank0 | Train Loss: 4.6521 | Train Loss Unweighted: 0.9975 | Train Loss Components: Classification: 0.0048 | Classification_binary: 0.4840 | Regression_cos_phi: 0.1694 | Regression_energy: 0.1167 | Regression_eta: 0.0078 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0985 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.4840 | Regression_cos_phi: 1.5483 | Regression_energy: 0.6593 | Regression_eta: 0.0489 | Regression_pt: 0.9711 | Regression_sin_phi: 0.8805 | LR: 9.21e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4707s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0032 | max_backward: 0.0396 | max_optimizer_step: 0.0610 | max_tensorboard_logging: 0.1186 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2717.4 | uss_MB: 1562.8 | child_rss_MB: 22641.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:09:42,553] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:09:42,621] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:10:05,866] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:10:05,866] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:08,549] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:11:08,549] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:33,836] INFO : Step 39000/500000 rank0 | Train Loss: 1.4668 | Train Loss Unweighted: 0.9746 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.8963 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0321 | Regression_eta: 0.0049 | Regression_pt: 0.0309 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0231 | Classification_binary: 0.8963 | Regression_cos_phi: 0.0486 | Regression_energy: 0.1816 | Regression_eta: 0.0310 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0285 | LR: 9.57e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5135s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0028 | max_backward: 0.0396 | max_optimizer_step: 0.0650 | max_tensorboard_logging: 0.1435 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2721.9 | uss_MB: 1567.2 | child_rss_MB: 22656.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:12:12,029] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:12:12,030] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:15,272] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:13:15,272] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:26,721] INFO : Step 40000/500000 rank0 | Train Loss: 3.0005 | Train Loss Unweighted: 0.5849 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2504 | Regression_cos_phi: 0.1917 | Regression_energy: 0.0578 | Regression_eta: 0.0202 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0033 | Classification_binary: 0.2504 | Regression_cos_phi: 1.7521 | Regression_energy: 0.3268 | Regression_eta: 0.1269 | Regression_pt: 0.5056 | Regression_sin_phi: 0.0354 | LR: 9.94e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5221s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0031 | max_backward: 0.0403 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1316 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2722.2 | uss_MB: 1567.5 | child_rss_MB: 22653.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:13:26,723] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:13:26,790] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:13:26,790] INFO : Running validation on rank0 +rank0 [2026-11-14 05:13:26,790] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:13:26,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:13:26,915] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:13:27,032] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:13:27,140] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:13:27,208] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:13:27,276] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:13:27,345] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:13:27,414] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:13:27,484] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:13:27,598] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:13:27,702] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:13:27,808] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:13:27,875] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:13:27,942] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:13:28,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:13:28,081] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:13:28,148] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:13:28,258] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:13:28,264] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:13:28,266] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:13:28,269] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:13:28,271] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:13:28,273] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:13:28,273] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:13:28,275] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:13:28,277] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:13:28,280] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:13:28,281] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:13:28,283] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:13:28,284] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:13:28,286] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:13:28,288] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:13:28,290] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:13:29,757] INFO : VALIDATION | Step=40000/500000 | Train Loss=3.0005 | Valid Loss=9.0151 | Train Loss Unweighted=0.5849 | Valid Loss Unweighted=5.2768 | Stale=2 | ETA=863.1m +rank0 [2026-11-14 05:13:29,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:13:30,119] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:13:30,126] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:30,175] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:13:30,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:13:30,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:13:31,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:13:31,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:13:31,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:13:31,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:13:32,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:13:32,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:13:32,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:13:32,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:13:33,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:13:33,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:13:33,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:13:33,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:13:34,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:13:34,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:13:34,438] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:13:34,446] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:35,005] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:35,370] INFO : Plotted number of elements +rank0 [2026-11-14 05:13:36,665] INFO : Plotted jets +rank0 [2026-11-14 05:13:37,628] INFO : Plotted jet ratio +rank0 [2026-11-14 05:14:30,188] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:14:30,188] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:30,536] INFO : Step 41000/500000 rank0 | Train Loss: 1.1779 | Train Loss Unweighted: 0.5557 | Train Loss Components: Classification: 0.0097 | Classification_binary: 0.4637 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0353 | Regression_eta: 0.0026 | Regression_pt: 0.0356 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1222 | Classification_binary: 0.4637 | Regression_cos_phi: 0.0344 | Regression_energy: 0.1996 | Regression_eta: 0.0165 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0448 | LR: 1.03e-04 | DataLoad Time: 0.0029s | Model Forward Time: 0.4967s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0383 | max_loss: 0.0030 | max_backward: 0.0424 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1195 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2742.4 | uss_MB: 1587.8 | child_rss_MB: 22649.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:15:33,448] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:15:33,448] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:36,355] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:16:36,355] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:22,777] INFO : Step 42000/500000 rank0 | Train Loss: 2.0732 | Train Loss Unweighted: 0.9518 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.7669 | Regression_cos_phi: 0.0018 | Regression_energy: 0.0868 | Regression_eta: 0.0033 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0030 | Classification_binary: 0.7669 | Regression_cos_phi: 0.0162 | Regression_energy: 0.4904 | Regression_eta: 0.0207 | Regression_pt: 0.7562 | Regression_sin_phi: 0.0199 | LR: 1.07e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5335s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0389 | max_loss: 0.0030 | max_backward: 0.0424 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.0 | uss_MB: 1588.3 | child_rss_MB: 22640.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:17:22,779] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:17:22,850] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:17:39,720] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:17:39,720] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:43,110] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:18:43,110] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:15,807] INFO : Step 43000/500000 rank0 | Train Loss: 1.8252 | Train Loss Unweighted: 0.6663 | Train Loss Components: Classification: 0.0037 | Classification_binary: 0.4811 | Regression_cos_phi: 0.0072 | Regression_energy: 0.0766 | Regression_eta: 0.0114 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.4811 | Regression_cos_phi: 0.0662 | Regression_energy: 0.4329 | Regression_eta: 0.0716 | Regression_pt: 0.6208 | Regression_sin_phi: 0.1067 | LR: 1.11e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5149s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1324 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2746.5 | uss_MB: 1591.9 | child_rss_MB: 22663.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:19:46,533] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:19:46,533] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:49,560] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:20:49,560] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:08,370] INFO : Step 44000/500000 rank0 | Train Loss: 0.7894 | Train Loss Unweighted: 0.2774 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0351 | Regression_eta: 0.0059 | Regression_pt: 0.0355 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0076 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0381 | Regression_energy: 0.1981 | Regression_eta: 0.0372 | Regression_pt: 0.2961 | Regression_sin_phi: 0.0181 | LR: 1.15e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5359s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0035 | max_backward: 0.0419 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1221 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2746.8 | uss_MB: 1592.2 | child_rss_MB: 22660.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:21:08,372] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:21:08,441] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:21:53,065] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:21:53,065] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:56,315] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:22:56,315] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:23:01,231] INFO : Step 45000/500000 rank0 | Train Loss: 1.7121 | Train Loss Unweighted: 0.9486 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.8219 | Regression_cos_phi: 0.0035 | Regression_energy: 0.0612 | Regression_eta: 0.0043 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0086 | Classification_binary: 0.8219 | Regression_cos_phi: 0.0324 | Regression_energy: 0.3459 | Regression_eta: 0.0271 | Regression_pt: 0.4586 | Regression_sin_phi: 0.0176 | LR: 1.19e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5407s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0032 | max_backward: 0.0438 | max_optimizer_step: 0.0715 | max_tensorboard_logging: 0.1388 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2746.8 | uss_MB: 1592.2 | child_rss_MB: 22661.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:23:59,561] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:23:59,561] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:53,279] INFO : Step 46000/500000 rank0 | Train Loss: 1.2403 | Train Loss Unweighted: 0.7231 | Train Loss Components: Classification: 0.0012 | Classification_binary: 0.6401 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0332 | Regression_eta: 0.0054 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0149 | Classification_binary: 0.6401 | Regression_cos_phi: 0.0242 | Regression_energy: 0.1879 | Regression_eta: 0.0337 | Regression_pt: 0.3246 | Regression_sin_phi: 0.0150 | LR: 1.23e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5446s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0032 | max_backward: 0.0407 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1384 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.0 | uss_MB: 1588.4 | child_rss_MB: 22654.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:24:53,281] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 05:24:53,351] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:25:02,666] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 05:25:02,666] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:05,027] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 05:26:05,028] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:44,909] INFO : Step 47000/500000 rank0 | Train Loss: 0.7379 | Train Loss Unweighted: 0.2431 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0311 | Regression_eta: 0.0035 | Regression_pt: 0.0310 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0055 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0244 | Regression_energy: 0.1757 | Regression_eta: 0.0218 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0870 | LR: 1.27e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5003s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0398 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1200 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2745.8 | uss_MB: 1591.4 | child_rss_MB: 22653.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:27:08,046] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 05:27:08,047] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:11,109] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 05:28:11,109] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:37,102] INFO : Step 48000/500000 rank0 | Train Loss: 0.5182 | Train Loss Unweighted: 0.1492 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.0898 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0241 | Regression_eta: 0.0038 | Regression_pt: 0.0254 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0038 | Classification_binary: 0.0898 | Regression_cos_phi: 0.0286 | Regression_energy: 0.1363 | Regression_eta: 0.0236 | Regression_pt: 0.2118 | Regression_sin_phi: 0.0242 | LR: 1.31e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5520s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0028 | max_backward: 0.0411 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1697 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2745.8 | uss_MB: 1591.4 | child_rss_MB: 22664.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:28:37,104] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 05:28:37,186] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:29:14,527] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:29:14,527] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:17,441] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:30:17,442] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:29,774] INFO : Step 49000/500000 rank0 | Train Loss: 0.9632 | Train Loss Unweighted: 0.3099 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0103 | Regression_energy: 0.0392 | Regression_eta: 0.0090 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0010 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0944 | Regression_energy: 0.2212 | Regression_eta: 0.0566 | Regression_pt: 0.3206 | Regression_sin_phi: 0.0636 | LR: 1.36e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5212s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0032 | max_backward: 0.0404 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2745.8 | uss_MB: 1591.5 | child_rss_MB: 22657.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:31:20,840] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:31:20,840] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:22,195] INFO : Step 50000/500000 rank0 | Train Loss: 1.6111 | Train Loss Unweighted: 0.7154 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.5680 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0693 | Regression_eta: 0.0051 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0177 | Classification_binary: 0.5680 | Regression_cos_phi: 0.0246 | Regression_energy: 0.3915 | Regression_eta: 0.0323 | Regression_pt: 0.5419 | Regression_sin_phi: 0.0350 | LR: 1.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5255s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0028 | max_backward: 0.0422 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1307 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2742.6 | uss_MB: 1588.2 | child_rss_MB: 22654.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:32:22,197] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:32:22,274] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:32:22,275] INFO : Running validation on rank0 +rank0 [2026-11-14 05:32:22,275] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:32:22,275] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:32:22,394] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:32:22,501] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:32:22,604] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:32:22,672] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:32:22,737] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:32:22,804] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:32:22,874] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:32:22,942] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:32:23,054] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:32:23,159] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:32:23,263] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:32:23,331] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:32:23,400] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:32:23,469] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:32:23,538] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:32:23,606] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:32:23,717] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:32:23,724] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:32:23,726] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:32:23,727] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:32:23,730] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:32:23,732] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:32:23,733] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:32:23,735] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:32:23,736] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:32:23,739] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:32:23,741] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:32:23,742] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:32:23,743] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:32:23,745] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:32:23,747] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:32:23,749] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:32:25,189] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.6111 | Valid Loss=10.3299 | Train Loss Unweighted=0.7154 | Valid Loss Unweighted=6.1870 | Stale=3 | ETA=845.8m +rank0 [2026-11-14 05:32:25,190] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:32:25,599] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:32:25,613] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:25,660] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:32:26,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:32:26,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:32:26,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:32:26,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:32:27,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:32:27,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:32:27,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:32:27,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:32:28,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:32:28,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:32:28,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:32:28,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:32:29,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:32:29,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:32:29,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:32:29,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:32:29,860] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:32:29,868] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:30,404] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:30,771] INFO : Plotted number of elements +rank0 [2026-11-14 05:32:32,056] INFO : Plotted jets +rank0 [2026-11-14 05:32:33,020] INFO : Plotted jet ratio +rank0 [2026-11-14 05:32:33,464] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:32:33,464] INFO : Training completed. Total time on device 0: 94.113min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..90cb0dad38921041ff90fd9e2b2827d3473ed734 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/train.log.0 @@ -0,0 +1,997 @@ +rank0 [2026-11-14 03:58:23,969] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 03:58:23,969] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 03:58:23,970] INFO : Instantiating model +rank0 [2026-11-14 03:58:24,000] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 03:58:24,000] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 03:58:24,002] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 03:58:24,002] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 03:58:24,002] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 03:58:24,003] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 03:58:24,003] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,005] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 03:58:24,005] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,008] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 03:58:24,008] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,011] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 03:58:24,011] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,014] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 03:58:24,014] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,017] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 03:58:24,017] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,019] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 03:58:24,020] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 03:58:24,020] INFO : Initializing output DNNs +rank0 [2026-11-14 03:58:24,028] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,031] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:58:24,038] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-14 03:58:24,038] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 03:58:24,038] INFO : jagged_attention enabled=True +rank0 [2026-11-14 03:58:24,038] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 03:58:24,038] INFO : task_queries enabled=True +rank0 [2026-11-14 03:58:24,038] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 03:58:24,038] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 03:58:24,038] INFO : classification_query parameters: 256 +rank0 [2026-11-14 03:58:24,038] INFO : regression_query parameters: 256 +rank0 [2026-11-14 03:58:24,039] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 03:58:24,039] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 03:58:24,039] INFO : output_mode=elementwise +rank0 [2026-11-14 03:58:24,039] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 03:58:24,039] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 03:58:24,039] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 03:58:24,039] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 03:58:24,039] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 03:58:24,039] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 03:58:24,039] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 03:58:24,039] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 03:58:24,040] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 03:58:24,040] INFO : MLPF __init__ done +rank0 [2026-11-14 03:58:24,040] INFO : Instantiated model +rank0 [2026-11-14 03:58:24,043] INFO : Moving model to device rank=0 +rank0 [2026-11-14 03:58:24,054] INFO : Moved model to device rank=0 +rank0 [2026-11-14 03:58:24,055] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 03:58:24,158] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 03:58:24,158] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 03:58:24,159] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 03:58:24,159] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 03:58:24,159] INFO : Total parameters: 3910930 +rank0 [2026-11-14 03:58:24,162] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 03:58:24,164] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597 +rank0 [2026-11-14 03:58:24,164] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597 +rank0 [2026-11-14 03:58:24,164] INFO : Getting dataloaders +rank0 [2026-11-14 03:58:24,164] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:58:26,028] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,070] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:26,115] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:26,148] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,174] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:26,204] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,230] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,262] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,304] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:58:26,419] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:58:26,419] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 03:58:26,419] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 03:58:26,419] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 03:58:26,420] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:58:26,427] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,434] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,441] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,448] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,455] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,463] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,470] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,478] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,485] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,492] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:58:26,492] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 03:58:26,492] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 03:58:26,547] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:58:26,684] INFO : Got dataloaders +rank0 [2026-11-14 03:58:26,684] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 03:58:26,684] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 03:58:26,684] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 03:58:26,690] INFO : Creating train_iterator +rank0 [2026-11-14 03:58:26,690] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 03:58:26,715] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:58:26,860] INFO : Created train_iterator +rank0 [2026-11-14 03:58:39,547] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.46730902671814, 'Classification': 0.5930971106886864, 'Regression_pt': 0.8945067819952964, 'Regression_eta': 1.1884842801094055, 'Regression_sin_phi': 0.8354161743819714, 'Regression_cos_phi': 0.8169610878825188, 'Regression_energy': 1.3217136013507842}, weights={'Classification_binary': 1.0, 'Classification': 12.590364456176758, 'Regression_pt': 8.347962379455566, 'Regression_eta': 6.283052444458008, 'Regression_sin_phi': 8.938429832458496, 'Regression_cos_phi': 9.140348434448242, 'Regression_energy': 5.649717807769775} +rank0 [2026-11-14 03:59:31,269] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 03:59:31,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:00:20,218] INFO : Step 1000/500000 rank0 | Train Loss: 11.3437 | Train Loss Unweighted: 5.1166 | Train Loss Components: Classification: 0.1328 | Classification_binary: 4.2277 | Regression_cos_phi: 0.0391 | Regression_energy: 0.3094 | Regression_eta: 0.0478 | Regression_pt: 0.3012 | Regression_sin_phi: 0.0586 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 1.6726 | Classification_binary: 4.2277 | Regression_cos_phi: 0.3575 | Regression_energy: 1.7478 | Regression_eta: 0.3005 | Regression_pt: 2.5144 | Regression_sin_phi: 0.5234 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5028s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0032 | max_backward: 0.0410 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1332 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2566.3 | uss_MB: 1397.9 | child_rss_MB: 22698.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:00:33,944] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:00:33,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:36,334] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:01:36,334] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:02:11,716] INFO : Step 2000/500000 rank0 | Train Loss: 8.7946 | Train Loss Unweighted: 4.4529 | Train Loss Components: Classification: 0.0430 | Classification_binary: 3.7877 | Regression_cos_phi: 0.0229 | Regression_energy: 0.2483 | Regression_eta: 0.0441 | Regression_pt: 0.2834 | Regression_sin_phi: 0.0235 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.5419 | Classification_binary: 3.7877 | Regression_cos_phi: 0.2094 | Regression_energy: 1.4027 | Regression_eta: 0.2772 | Regression_pt: 2.3655 | Regression_sin_phi: 0.2104 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5131s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0379 | max_loss: 0.0032 | max_backward: 0.0400 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.1266 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2567.1 | uss_MB: 1398.7 | child_rss_MB: 22627.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:02:11,719] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:02:39,392] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:02:39,392] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:41,799] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:03:41,800] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:03,365] INFO : Step 3000/500000 rank0 | Train Loss: 6.0888 | Train Loss Unweighted: 3.2458 | Train Loss Components: Classification: 0.0324 | Classification_binary: 2.8118 | Regression_cos_phi: 0.0236 | Regression_energy: 0.1741 | Regression_eta: 0.0218 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0217 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.4074 | Classification_binary: 2.8118 | Regression_cos_phi: 0.2157 | Regression_energy: 0.9834 | Regression_eta: 0.1371 | Regression_pt: 1.3397 | Regression_sin_phi: 0.1937 | LR: 2.05e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4809s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0343 | max_loss: 0.0026 | max_backward: 0.0404 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1198 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.2 | uss_MB: 1402.2 | child_rss_MB: 22628.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:04:44,761] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:04:44,762] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:47,051] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:05:47,051] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:54,755] INFO : Step 4000/500000 rank0 | Train Loss: 8.1598 | Train Loss Unweighted: 3.2610 | Train Loss Components: Classification: 0.0341 | Classification_binary: 2.4828 | Regression_cos_phi: 0.0075 | Regression_energy: 0.3527 | Regression_eta: 0.0115 | Regression_pt: 0.3640 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.4293 | Classification_binary: 2.4828 | Regression_cos_phi: 0.0685 | Regression_energy: 1.9927 | Regression_eta: 0.0720 | Regression_pt: 3.0383 | Regression_sin_phi: 0.0762 | LR: 2.08e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4972s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0031 | max_backward: 0.0405 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1230 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.2 | uss_MB: 1402.3 | child_rss_MB: 22627.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:05:54,757] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:06:49,674] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:06:49,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:45,989] INFO : Step 5000/500000 rank0 | Train Loss: 5.2044 | Train Loss Unweighted: 3.1028 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.7782 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1327 | Regression_eta: 0.0138 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.2820 | Classification_binary: 2.7782 | Regression_cos_phi: 0.0500 | Regression_energy: 0.7497 | Regression_eta: 0.0870 | Regression_pt: 1.1945 | Regression_sin_phi: 0.0630 | LR: 2.13e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1194 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2599.0 | uss_MB: 1431.1 | child_rss_MB: 22622.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:07:52,381] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:07:52,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:54,354] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:08:54,354] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:36,869] INFO : Step 6000/500000 rank0 | Train Loss: 6.5604 | Train Loss Unweighted: 3.3120 | Train Loss Components: Classification: 0.0257 | Classification_binary: 2.8114 | Regression_cos_phi: 0.0525 | Regression_energy: 0.1764 | Regression_eta: 0.0554 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0164 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.3234 | Classification_binary: 2.8114 | Regression_cos_phi: 0.4801 | Regression_energy: 0.9968 | Regression_eta: 0.3479 | Regression_pt: 1.4545 | Regression_sin_phi: 0.1463 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5239s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0386 | max_loss: 0.0030 | max_backward: 0.0411 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1215 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2599.0 | uss_MB: 1431.1 | child_rss_MB: 22620.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:09:36,871] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:09:57,142] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:09:57,142] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:59,777] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:10:59,777] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:27,518] INFO : Step 7000/500000 rank0 | Train Loss: 7.4770 | Train Loss Unweighted: 3.3970 | Train Loss Components: Classification: 0.0110 | Classification_binary: 2.7294 | Regression_cos_phi: 0.0063 | Regression_energy: 0.3194 | Regression_eta: 0.0095 | Regression_pt: 0.3144 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1384 | Classification_binary: 2.7294 | Regression_cos_phi: 0.0574 | Regression_energy: 1.8047 | Regression_eta: 0.0597 | Regression_pt: 2.6243 | Regression_sin_phi: 0.0631 | LR: 2.26e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4591s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0026 | max_backward: 0.0376 | max_optimizer_step: 0.0589 | max_tensorboard_logging: 0.1272 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2599.1 | uss_MB: 1431.2 | child_rss_MB: 22620.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:12:01,863] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:12:01,863] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:05,518] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:13:05,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:20,171] INFO : Step 8000/500000 rank0 | Train Loss: 5.2526 | Train Loss Unweighted: 2.8218 | Train Loss Components: Classification: 0.0304 | Classification_binary: 2.4563 | Regression_cos_phi: 0.0130 | Regression_energy: 0.1423 | Regression_eta: 0.0063 | Regression_pt: 0.1681 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.3822 | Classification_binary: 2.4563 | Regression_cos_phi: 0.1184 | Regression_energy: 0.8040 | Regression_eta: 0.0395 | Regression_pt: 1.4034 | Regression_sin_phi: 0.0487 | LR: 2.34e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4568s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0356 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0597 | max_tensorboard_logging: 0.1197 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2599.2 | uss_MB: 1431.3 | child_rss_MB: 22612.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 20.8 +rank0 [2026-11-14 04:13:20,172] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:13:20,240] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:14:06,597] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:14:06,597] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:09,690] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:15:09,690] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:10,978] INFO : Step 9000/500000 rank0 | Train Loss: 5.0189 | Train Loss Unweighted: 3.1561 | Train Loss Components: Classification: 0.0103 | Classification_binary: 2.8590 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1326 | Regression_eta: 0.0095 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1298 | Classification_binary: 2.8590 | Regression_cos_phi: 0.0953 | Regression_energy: 0.7490 | Regression_eta: 0.0599 | Regression_pt: 1.0491 | Regression_sin_phi: 0.0769 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5086s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0026 | max_backward: 0.0406 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.1366 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2567.8 | uss_MB: 1400.0 | child_rss_MB: 22596.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:16:12,538] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:16:12,538] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:17:03,070] INFO : Step 10000/500000 rank0 | Train Loss: 5.2751 | Train Loss Unweighted: 2.4615 | Train Loss Components: Classification: 0.0069 | Classification_binary: 2.0719 | Regression_cos_phi: 0.1111 | Regression_energy: 0.0738 | Regression_eta: 0.0124 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0992 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0864 | Classification_binary: 2.0719 | Regression_cos_phi: 1.0151 | Regression_energy: 0.4169 | Regression_eta: 0.0782 | Regression_pt: 0.7195 | Regression_sin_phi: 0.8871 | LR: 2.52e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5344s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0031 | max_backward: 0.0412 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1416 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2568.5 | uss_MB: 1400.5 | child_rss_MB: 22617.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:17:03,072] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:17:03,154] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:17:03,154] INFO : Running validation on rank0 +rank0 [2026-11-14 04:17:03,154] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:17:03,154] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:17:03,354] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:17:03,564] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:17:03,634] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:17:03,690] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:17:03,746] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:17:03,796] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:17:03,852] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:17:03,910] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:17:03,959] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:17:04,029] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:17:04,100] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:17:04,156] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:17:04,214] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:17:04,264] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:17:04,322] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:17:04,380] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:17:04,431] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:17:04,436] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:17:04,438] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:17:04,440] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:17:04,441] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:17:04,444] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:17:04,445] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:17:04,447] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:17:04,449] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:17:04,451] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:17:04,453] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:17:04,455] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:17:04,455] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:17:04,456] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:17:04,459] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:17:04,462] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:17:05,985] INFO : VALIDATION | Step=10000/500000 | Train Loss=5.2751 | Valid Loss=6.3093 | Train Loss Unweighted=2.4615 | Valid Loss Unweighted=3.1808 | Stale=0 | ETA=914.1m +rank0 [2026-11-14 04:17:05,986] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:17:06,348] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:17:06,356] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:06,404] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:17:07,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:17:07,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:17:07,681] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:17:07,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:17:08,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:17:08,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:17:08,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:17:08,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:17:09,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:17:09,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:17:09,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:17:09,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:17:10,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:17:10,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:17:10,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:17:10,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:17:10,897] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:17:10,904] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:11,445] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:12,337] INFO : Plotted number of elements +rank0 [2026-11-14 04:17:13,673] INFO : Plotted jets +rank0 [2026-11-14 04:17:14,613] INFO : Plotted jet ratio +rank0 [2026-11-14 04:17:27,899] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:17:27,899] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:30,148] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:18:30,148] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:06,357] INFO : Step 11000/500000 rank0 | Train Loss: 6.0337 | Train Loss Unweighted: 3.5951 | Train Loss Components: Classification: 0.0133 | Classification_binary: 3.2019 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1824 | Regression_eta: 0.0111 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1672 | Classification_binary: 3.2019 | Regression_cos_phi: 0.0573 | Regression_energy: 1.0304 | Regression_eta: 0.0697 | Regression_pt: 1.4595 | Regression_sin_phi: 0.0477 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5149s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0412 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2700.4 | uss_MB: 1545.7 | child_rss_MB: 22627.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:19:33,022] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:19:33,022] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:35,509] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:20:35,509] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:57,908] INFO : Step 12000/500000 rank0 | Train Loss: 3.2011 | Train Loss Unweighted: 1.6724 | Train Loss Components: Classification: 0.0254 | Classification_binary: 1.4470 | Regression_cos_phi: 0.0064 | Regression_energy: 0.0805 | Regression_eta: 0.0128 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.3194 | Classification_binary: 1.4470 | Regression_cos_phi: 0.0583 | Regression_energy: 0.4547 | Regression_eta: 0.0806 | Regression_pt: 0.7802 | Regression_sin_phi: 0.0608 | LR: 2.75e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4982s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0398 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1192 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2700.4 | uss_MB: 1545.7 | child_rss_MB: 22629.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:20:57,910] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:20:57,982] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:21:38,435] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:21:38,435] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:41,333] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:22:41,334] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:49,980] INFO : Step 13000/500000 rank0 | Train Loss: 5.6114 | Train Loss Unweighted: 2.7311 | Train Loss Components: Classification: 0.0139 | Classification_binary: 2.2691 | Regression_cos_phi: 0.0035 | Regression_energy: 0.2057 | Regression_eta: 0.0122 | Regression_pt: 0.2198 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1750 | Classification_binary: 2.2691 | Regression_cos_phi: 0.0323 | Regression_energy: 1.1624 | Regression_eta: 0.0767 | Regression_pt: 1.8345 | Regression_sin_phi: 0.0614 | LR: 2.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5342s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0032 | max_backward: 0.0415 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1258 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2653.7 | uss_MB: 1499.0 | child_rss_MB: 22625.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:23:44,172] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:23:44,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:41,324] INFO : Step 14000/500000 rank0 | Train Loss: 7.0186 | Train Loss Unweighted: 3.4251 | Train Loss Components: Classification: 0.0060 | Classification_binary: 2.8562 | Regression_cos_phi: 0.0680 | Regression_energy: 0.2337 | Regression_eta: 0.0202 | Regression_pt: 0.2306 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 2.8562 | Regression_cos_phi: 0.6219 | Regression_energy: 1.3204 | Regression_eta: 0.1270 | Regression_pt: 1.9253 | Regression_sin_phi: 0.0922 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5108s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0399 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1267 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2657.9 | uss_MB: 1503.1 | child_rss_MB: 22625.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:41,326] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:24:41,400] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:24:46,944] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:24:46,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:49,210] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:25:49,210] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:32,920] INFO : Step 15000/500000 rank0 | Train Loss: 5.9356 | Train Loss Unweighted: 2.6748 | Train Loss Components: Classification: 0.0145 | Classification_binary: 2.1700 | Regression_cos_phi: 0.0353 | Regression_energy: 0.2035 | Regression_eta: 0.0073 | Regression_pt: 0.2002 | Regression_sin_phi: 0.0439 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1830 | Classification_binary: 2.1700 | Regression_cos_phi: 0.3229 | Regression_energy: 1.1496 | Regression_eta: 0.0460 | Regression_pt: 1.6716 | Regression_sin_phi: 0.3924 | LR: 3.17e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4928s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1302 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2660.5 | uss_MB: 1505.8 | child_rss_MB: 22623.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:26:52,226] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:26:52,226] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:54,328] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:27:54,329] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:23,866] INFO : Step 16000/500000 rank0 | Train Loss: 5.8052 | Train Loss Unweighted: 2.7445 | Train Loss Components: Classification: 0.0102 | Classification_binary: 2.2459 | Regression_cos_phi: 0.0067 | Regression_energy: 0.2360 | Regression_eta: 0.0101 | Regression_pt: 0.2248 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1287 | Classification_binary: 2.2459 | Regression_cos_phi: 0.0617 | Regression_energy: 1.3333 | Regression_eta: 0.0633 | Regression_pt: 1.8762 | Regression_sin_phi: 0.0961 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5077s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0356 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0710 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2660.6 | uss_MB: 1505.9 | child_rss_MB: 22618.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:28:23,868] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:23,937] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:28:56,703] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:28:56,703] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:59,314] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:29:59,314] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:15,158] INFO : Step 17000/500000 rank0 | Train Loss: 8.2892 | Train Loss Unweighted: 2.6973 | Train Loss Components: Classification: 0.0141 | Classification_binary: 1.7826 | Regression_cos_phi: 0.0049 | Regression_energy: 0.4385 | Regression_eta: 0.0059 | Regression_pt: 0.4478 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1769 | Classification_binary: 1.7826 | Regression_cos_phi: 0.0446 | Regression_energy: 2.4772 | Regression_eta: 0.0372 | Regression_pt: 3.7383 | Regression_sin_phi: 0.0323 | LR: 3.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4965s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0026 | max_backward: 0.0393 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1238 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2657.0 | uss_MB: 1502.3 | child_rss_MB: 22604.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:31:02,130] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:31:02,130] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:04,596] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:32:04,596] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:06,773] INFO : Step 18000/500000 rank0 | Train Loss: 7.0760 | Train Loss Unweighted: 2.9533 | Train Loss Components: Classification: 0.0081 | Classification_binary: 2.2810 | Regression_cos_phi: 0.0107 | Regression_energy: 0.3111 | Regression_eta: 0.0110 | Regression_pt: 0.3276 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1021 | Classification_binary: 2.2810 | Regression_cos_phi: 0.0981 | Regression_energy: 1.7579 | Regression_eta: 0.0693 | Regression_pt: 2.7346 | Regression_sin_phi: 0.0330 | LR: 3.69e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4820s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0029 | max_backward: 0.0413 | max_optimizer_step: 0.0638 | max_tensorboard_logging: 0.1193 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2657.7 | uss_MB: 1503.0 | child_rss_MB: 22630.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:32:06,775] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:32:06,844] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:33:07,124] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:33:07,125] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:57,652] INFO : Step 19000/500000 rank0 | Train Loss: 5.0555 | Train Loss Unweighted: 2.4955 | Train Loss Components: Classification: 0.0098 | Classification_binary: 2.0867 | Regression_cos_phi: 0.0126 | Regression_energy: 0.1742 | Regression_eta: 0.0150 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1232 | Classification_binary: 2.0867 | Regression_cos_phi: 0.1151 | Regression_energy: 0.9841 | Regression_eta: 0.0945 | Regression_pt: 1.5717 | Regression_sin_phi: 0.0802 | LR: 3.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5119s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0381 | max_loss: 0.0031 | max_backward: 0.0416 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1247 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2660.4 | uss_MB: 1505.7 | child_rss_MB: 22624.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:34:09,634] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:34:09,634] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:12,367] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:35:12,367] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:50,160] INFO : Step 20000/500000 rank0 | Train Loss: 2.9175 | Train Loss Unweighted: 1.6369 | Train Loss Components: Classification: 0.0085 | Classification_binary: 1.4348 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0878 | Regression_eta: 0.0041 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1067 | Classification_binary: 1.4348 | Regression_cos_phi: 0.0340 | Regression_energy: 0.4960 | Regression_eta: 0.0257 | Regression_pt: 0.7854 | Regression_sin_phi: 0.0349 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0033 | max_backward: 0.0404 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1322 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2660.8 | uss_MB: 1506.0 | child_rss_MB: 22630.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:50,162] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:35:50,229] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:35:50,230] INFO : Running validation on rank0 +rank0 [2026-11-14 04:35:50,230] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:35:50,230] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:35:50,352] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:35:50,462] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:35:50,566] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:35:50,636] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:35:50,704] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:35:50,771] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:35:50,841] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:35:50,911] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:35:51,012] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:35:51,117] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:35:51,219] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:35:51,287] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:35:51,355] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:35:51,420] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:35:51,489] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:35:51,558] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:35:51,659] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:35:51,665] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:35:51,667] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:35:51,669] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:35:51,671] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:35:51,673] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:35:51,675] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:35:51,677] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:35:51,679] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:35:51,681] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:35:51,683] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:35:51,685] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:35:51,686] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:35:51,687] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:35:51,689] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:35:51,692] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:35:53,209] INFO : VALIDATION | Step=20000/500000 | Train Loss=2.9175 | Valid Loss=5.8167 | Train Loss Unweighted=1.6369 | Valid Loss Unweighted=2.9479 | Stale=0 | ETA=898.6m +rank0 [2026-11-14 04:35:53,210] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:35:53,611] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:35:53,619] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:53,659] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:35:54,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:35:54,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:35:54,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:35:54,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:35:55,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:35:55,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:35:55,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:35:55,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:35:56,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:35:56,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:35:56,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:35:56,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:35:57,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:35:57,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:35:57,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:35:57,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:35:57,854] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:35:57,861] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:58,398] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:35:58,769] INFO : Plotted number of elements +rank0 [2026-11-14 04:36:00,060] INFO : Plotted jets +rank0 [2026-11-14 04:36:01,026] INFO : Plotted jet ratio +rank0 [2026-11-14 04:36:27,410] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:36:27,410] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:30,216] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:37:30,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:53,486] INFO : Step 21000/500000 rank0 | Train Loss: 3.1722 | Train Loss Unweighted: 1.8618 | Train Loss Components: Classification: 0.0017 | Classification_binary: 1.6532 | Regression_cos_phi: 0.0100 | Regression_energy: 0.0866 | Regression_eta: 0.0047 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0214 | Classification_binary: 1.6532 | Regression_cos_phi: 0.0915 | Regression_energy: 0.4893 | Regression_eta: 0.0295 | Regression_pt: 0.7967 | Regression_sin_phi: 0.0906 | LR: 4.28e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5348s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0029 | max_backward: 0.0396 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1420 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2691.4 | uss_MB: 1536.7 | child_rss_MB: 22633.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:38:32,977] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:38:32,977] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:34,349] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:39:34,349] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:43,476] INFO : Step 22000/500000 rank0 | Train Loss: 3.7398 | Train Loss Unweighted: 1.6836 | Train Loss Components: Classification: 0.0134 | Classification_binary: 1.3556 | Regression_cos_phi: 0.0030 | Regression_energy: 0.1496 | Regression_eta: 0.0056 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1690 | Classification_binary: 1.3556 | Regression_cos_phi: 0.0270 | Regression_energy: 0.8452 | Regression_eta: 0.0351 | Regression_pt: 1.2755 | Regression_sin_phi: 0.0323 | LR: 4.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4722s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0026 | max_backward: 0.0380 | max_optimizer_step: 0.0596 | max_tensorboard_logging: 0.1254 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2692.4 | uss_MB: 1537.8 | child_rss_MB: 22634.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:39:43,478] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:39:43,548] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:40:38,525] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:40:38,525] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:36,511] INFO : Step 23000/500000 rank0 | Train Loss: 3.4357 | Train Loss Unweighted: 2.2239 | Train Loss Components: Classification: 0.0094 | Classification_binary: 2.0360 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0754 | Regression_eta: 0.0081 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1189 | Classification_binary: 2.0360 | Regression_cos_phi: 0.0664 | Regression_energy: 0.4263 | Regression_eta: 0.0511 | Regression_pt: 0.6543 | Regression_sin_phi: 0.0827 | LR: 4.73e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0028 | max_backward: 0.0389 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1292 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2695.9 | uss_MB: 1541.2 | child_rss_MB: 22636.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:41:41,009] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:41:41,009] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:43,005] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:42:43,005] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:27,853] INFO : Step 24000/500000 rank0 | Train Loss: 5.0597 | Train Loss Unweighted: 2.2145 | Train Loss Components: Classification: 0.0036 | Classification_binary: 1.7285 | Regression_cos_phi: 0.0136 | Regression_energy: 0.1123 | Regression_eta: 0.2218 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0154 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 1.7285 | Regression_cos_phi: 0.1239 | Regression_energy: 0.6342 | Regression_eta: 1.3936 | Regression_pt: 0.9960 | Regression_sin_phi: 0.1377 | LR: 4.97e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5374s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1492 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2695.9 | uss_MB: 1541.2 | child_rss_MB: 22631.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:43:27,855] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:43:27,927] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:43:46,437] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:43:46,437] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:49,305] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:44:49,305] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:20,116] INFO : Step 25000/500000 rank0 | Train Loss: 2.9067 | Train Loss Unweighted: 1.3899 | Train Loss Components: Classification: 0.0039 | Classification_binary: 1.1419 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1153 | Regression_eta: 0.0072 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 1.1419 | Regression_cos_phi: 0.0306 | Regression_energy: 0.6516 | Regression_eta: 0.0453 | Regression_pt: 0.9646 | Regression_sin_phi: 0.0231 | LR: 5.22e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5109s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0027 | max_backward: 0.0390 | max_optimizer_step: 0.0666 | max_tensorboard_logging: 0.1327 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2691.5 | uss_MB: 1536.8 | child_rss_MB: 22620.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:45:52,240] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:45:52,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:54,576] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:46:54,576] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:11,370] INFO : Step 26000/500000 rank0 | Train Loss: 2.4612 | Train Loss Unweighted: 1.0077 | Train Loss Components: Classification: 0.0017 | Classification_binary: 0.7699 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1117 | Regression_eta: 0.0028 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0212 | Classification_binary: 0.7699 | Regression_cos_phi: 0.0559 | Regression_energy: 0.6310 | Regression_eta: 0.0177 | Regression_pt: 0.9442 | Regression_sin_phi: 0.0211 | LR: 5.47e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5176s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0027 | max_backward: 0.0405 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1322 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2691.9 | uss_MB: 1537.2 | child_rss_MB: 22628.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:47:11,372] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:47:11,442] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:47:57,172] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:47:57,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:59,107] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:48:59,107] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:02,164] INFO : Step 27000/500000 rank0 | Train Loss: 4.3758 | Train Loss Unweighted: 2.2342 | Train Loss Components: Classification: 0.0129 | Classification_binary: 1.8957 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1475 | Regression_eta: 0.0076 | Regression_pt: 0.1509 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1620 | Classification_binary: 1.8957 | Regression_cos_phi: 0.0773 | Regression_energy: 0.8332 | Regression_eta: 0.0478 | Regression_pt: 1.2600 | Regression_sin_phi: 0.0998 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5058s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0364 | max_loss: 0.0031 | max_backward: 0.0395 | max_optimizer_step: 0.0646 | max_tensorboard_logging: 0.1267 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2695.4 | uss_MB: 1540.7 | child_rss_MB: 22640.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:50:02,011] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:50:02,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:53,272] INFO : Step 28000/500000 rank0 | Train Loss: 2.3043 | Train Loss Unweighted: 1.4392 | Train Loss Components: Classification: 0.0110 | Classification_binary: 1.3068 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0536 | Regression_eta: 0.0085 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1389 | Classification_binary: 1.3068 | Regression_cos_phi: 0.0457 | Regression_energy: 0.3028 | Regression_eta: 0.0532 | Regression_pt: 0.4015 | Regression_sin_phi: 0.0554 | LR: 6.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5034s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0031 | max_backward: 0.0395 | max_optimizer_step: 0.0661 | max_tensorboard_logging: 0.1346 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2695.6 | uss_MB: 1540.9 | child_rss_MB: 22637.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:50:53,274] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:50:53,343] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:51:04,288] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:51:04,288] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:06,383] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:52:06,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:44,301] INFO : Step 29000/500000 rank0 | Train Loss: 2.5663 | Train Loss Unweighted: 1.8859 | Train Loss Components: Classification: 0.0058 | Classification_binary: 1.7797 | Regression_cos_phi: 0.0024 | Regression_energy: 0.0395 | Regression_eta: 0.0106 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 1.7797 | Regression_cos_phi: 0.0221 | Regression_energy: 0.2230 | Regression_eta: 0.0664 | Regression_pt: 0.3715 | Regression_sin_phi: 0.0307 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4996s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0035 | max_backward: 0.0400 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2696.5 | uss_MB: 1541.9 | child_rss_MB: 22636.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:53:09,264] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:53:09,264] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:12,213] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:54:12,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:36,449] INFO : Step 30000/500000 rank0 | Train Loss: 3.1678 | Train Loss Unweighted: 2.0514 | Train Loss Components: Classification: 0.0096 | Classification_binary: 1.8763 | Regression_cos_phi: 0.0059 | Regression_energy: 0.0759 | Regression_eta: 0.0064 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1203 | Classification_binary: 1.8763 | Regression_cos_phi: 0.0541 | Regression_energy: 0.4290 | Regression_eta: 0.0402 | Regression_pt: 0.6096 | Regression_sin_phi: 0.0384 | LR: 6.58e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5166s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0388 | max_loss: 0.0031 | max_backward: 0.0405 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1238 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2692.2 | uss_MB: 1537.5 | child_rss_MB: 22640.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:54:36,451] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:54:36,521] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:54:36,521] INFO : Running validation on rank0 +rank0 [2026-11-14 04:54:36,521] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:54:36,521] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:54:36,644] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:54:36,752] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:54:36,856] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:54:36,925] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:54:36,991] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:54:37,056] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:54:37,122] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:54:37,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:54:37,293] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:54:37,394] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:54:37,494] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:54:37,564] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:54:37,632] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:54:37,701] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:54:37,768] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:54:37,837] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:54:37,936] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:54:37,942] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:54:37,945] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:54:37,947] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:54:37,949] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:54:37,951] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:54:37,952] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:54:37,953] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:54:37,955] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:54:37,957] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:54:37,959] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:54:37,961] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:54:37,962] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:54:37,964] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:54:37,964] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:54:37,966] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:54:39,413] INFO : VALIDATION | Step=30000/500000 | Train Loss=3.1678 | Valid Loss=6.6356 | Train Loss Unweighted=2.0514 | Valid Loss Unweighted=3.2661 | Stale=1 | ETA=880.7m +rank0 [2026-11-14 04:54:39,413] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:54:39,798] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:54:39,805] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:39,854] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:54:40,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:54:40,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:54:40,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:54:41,095] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:54:41,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:54:41,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:54:41,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:54:42,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:54:42,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:54:42,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:54:42,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:54:43,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:54:43,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:54:43,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:54:43,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:54:44,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:54:44,066] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:54:44,073] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:44,613] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:44,967] INFO : Plotted number of elements +rank0 [2026-11-14 04:54:46,251] INFO : Plotted jets +rank0 [2026-11-14 04:54:47,187] INFO : Plotted jet ratio +rank0 [2026-11-14 04:55:26,573] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:55:26,573] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:29,349] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:56:29,349] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:39,768] INFO : Step 31000/500000 rank0 | Train Loss: 1.9812 | Train Loss Unweighted: 1.3334 | Train Loss Components: Classification: 0.0014 | Classification_binary: 1.2307 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0405 | Regression_eta: 0.0057 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0180 | Classification_binary: 1.2307 | Regression_cos_phi: 0.0549 | Regression_energy: 0.2290 | Regression_eta: 0.0358 | Regression_pt: 0.3577 | Regression_sin_phi: 0.0550 | LR: 6.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5170s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0030 | max_backward: 0.0410 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2800.6 | uss_MB: 1645.9 | child_rss_MB: 22641.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:57:32,176] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:57:32,176] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:31,392] INFO : Step 32000/500000 rank0 | Train Loss: 4.0279 | Train Loss Unweighted: 1.9769 | Train Loss Components: Classification: 0.0084 | Classification_binary: 1.6465 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1506 | Regression_eta: 0.0084 | Regression_pt: 0.1448 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1054 | Classification_binary: 1.6465 | Regression_cos_phi: 0.0555 | Regression_energy: 0.8510 | Regression_eta: 0.0531 | Regression_pt: 1.2087 | Regression_sin_phi: 0.1076 | LR: 7.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5358s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0032 | max_backward: 0.0403 | max_optimizer_step: 0.0666 | max_tensorboard_logging: 0.1405 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2800.6 | uss_MB: 1645.9 | child_rss_MB: 22640.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:58:31,394] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:58:31,464] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:58:35,231] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:58:35,231] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:38,137] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:59:38,137] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:23,359] INFO : Step 33000/500000 rank0 | Train Loss: 3.6434 | Train Loss Unweighted: 1.1686 | Train Loss Components: Classification: 0.0012 | Classification_binary: 0.8319 | Regression_cos_phi: 0.1878 | Regression_energy: 0.0560 | Regression_eta: 0.0094 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0295 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0145 | Classification_binary: 0.8319 | Regression_cos_phi: 1.7165 | Regression_energy: 0.3166 | Regression_eta: 0.0590 | Regression_pt: 0.4413 | Regression_sin_phi: 0.2637 | LR: 7.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5344s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0031 | max_backward: 0.0401 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1381 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2716.9 | uss_MB: 1562.2 | child_rss_MB: 22637.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:00:40,837] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:00:40,837] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:43,478] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:01:43,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:15,102] INFO : Step 34000/500000 rank0 | Train Loss: 1.7795 | Train Loss Unweighted: 1.0908 | Train Loss Components: Classification: 0.0029 | Classification_binary: 0.9815 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0454 | Regression_eta: 0.0060 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.9815 | Regression_cos_phi: 0.0609 | Regression_energy: 0.2562 | Regression_eta: 0.0374 | Regression_pt: 0.3631 | Regression_sin_phi: 0.0441 | LR: 7.83e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5078s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2717.6 | uss_MB: 1563.0 | child_rss_MB: 22624.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:02:15,104] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:02:15,174] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:02:46,790] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:02:46,790] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:49,310] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:03:49,310] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:07,424] INFO : Step 35000/500000 rank0 | Train Loss: 3.4663 | Train Loss Unweighted: 1.1965 | Train Loss Components: Classification: 0.0052 | Classification_binary: 0.8256 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1719 | Regression_eta: 0.0114 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.8256 | Regression_cos_phi: 0.0667 | Regression_energy: 0.9710 | Regression_eta: 0.0719 | Regression_pt: 1.4049 | Regression_sin_phi: 0.0605 | LR: 8.16e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5081s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0389 | max_optimizer_step: 0.0629 | max_tensorboard_logging: 0.1403 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2722.1 | uss_MB: 1567.4 | child_rss_MB: 22649.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:04:52,435] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:04:52,435] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:55,124] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:05:55,125] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:59,133] INFO : Step 36000/500000 rank0 | Train Loss: 1.3828 | Train Loss Unweighted: 0.8913 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.8113 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0344 | Regression_eta: 0.0054 | Regression_pt: 0.0332 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0091 | Classification_binary: 0.8113 | Regression_cos_phi: 0.0361 | Regression_energy: 0.1941 | Regression_eta: 0.0338 | Regression_pt: 0.2773 | Regression_sin_phi: 0.0212 | LR: 8.50e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5236s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0031 | max_backward: 0.0413 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1334 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2722.1 | uss_MB: 1567.4 | child_rss_MB: 22647.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:05:59,135] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:05:59,205] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:06:58,352] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:06:58,352] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:49,854] INFO : Step 37000/500000 rank0 | Train Loss: 1.6053 | Train Loss Unweighted: 0.9423 | Train Loss Components: Classification: 0.0054 | Classification_binary: 0.8389 | Regression_cos_phi: 0.0090 | Regression_energy: 0.0343 | Regression_eta: 0.0185 | Regression_pt: 0.0297 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.8389 | Regression_cos_phi: 0.0825 | Regression_energy: 0.1937 | Regression_eta: 0.1164 | Regression_pt: 0.2478 | Regression_sin_phi: 0.0585 | LR: 8.85e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4522s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0337 | max_loss: 0.0026 | max_backward: 0.0378 | max_optimizer_step: 0.0593 | max_tensorboard_logging: 0.1166 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2722.1 | uss_MB: 1567.5 | child_rss_MB: 22647.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:07:59,603] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:07:59,603] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:04,453] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:09:04,453] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:42,551] INFO : Step 38000/500000 rank0 | Train Loss: 4.6521 | Train Loss Unweighted: 0.9975 | Train Loss Components: Classification: 0.0048 | Classification_binary: 0.4840 | Regression_cos_phi: 0.1694 | Regression_energy: 0.1167 | Regression_eta: 0.0078 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0985 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.4840 | Regression_cos_phi: 1.5483 | Regression_energy: 0.6593 | Regression_eta: 0.0489 | Regression_pt: 0.9711 | Regression_sin_phi: 0.8805 | LR: 9.21e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4707s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0032 | max_backward: 0.0396 | max_optimizer_step: 0.0610 | max_tensorboard_logging: 0.1186 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2717.4 | uss_MB: 1562.8 | child_rss_MB: 22641.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:09:42,553] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:09:42,621] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:10:05,866] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:10:05,866] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:08,549] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:11:08,549] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:33,836] INFO : Step 39000/500000 rank0 | Train Loss: 1.4668 | Train Loss Unweighted: 0.9746 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.8963 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0321 | Regression_eta: 0.0049 | Regression_pt: 0.0309 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0231 | Classification_binary: 0.8963 | Regression_cos_phi: 0.0486 | Regression_energy: 0.1816 | Regression_eta: 0.0310 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0285 | LR: 9.57e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5135s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0028 | max_backward: 0.0396 | max_optimizer_step: 0.0650 | max_tensorboard_logging: 0.1435 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2721.9 | uss_MB: 1567.2 | child_rss_MB: 22656.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:12:12,029] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:12:12,030] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:15,272] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:13:15,272] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:26,721] INFO : Step 40000/500000 rank0 | Train Loss: 3.0005 | Train Loss Unweighted: 0.5849 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.2504 | Regression_cos_phi: 0.1917 | Regression_energy: 0.0578 | Regression_eta: 0.0202 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0033 | Classification_binary: 0.2504 | Regression_cos_phi: 1.7521 | Regression_energy: 0.3268 | Regression_eta: 0.1269 | Regression_pt: 0.5056 | Regression_sin_phi: 0.0354 | LR: 9.94e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5221s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0031 | max_backward: 0.0403 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1316 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2722.2 | uss_MB: 1567.5 | child_rss_MB: 22653.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:13:26,723] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:13:26,790] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:13:26,790] INFO : Running validation on rank0 +rank0 [2026-11-14 05:13:26,790] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:13:26,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:13:26,915] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:13:27,032] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:13:27,140] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:13:27,208] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:13:27,276] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:13:27,345] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:13:27,414] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:13:27,484] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:13:27,598] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:13:27,702] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:13:27,808] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:13:27,875] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:13:27,942] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:13:28,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:13:28,081] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:13:28,148] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:13:28,258] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:13:28,264] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:13:28,266] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:13:28,269] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:13:28,271] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:13:28,273] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:13:28,273] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:13:28,275] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:13:28,277] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:13:28,280] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:13:28,281] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:13:28,283] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:13:28,284] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:13:28,286] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:13:28,288] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:13:28,290] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:13:29,757] INFO : VALIDATION | Step=40000/500000 | Train Loss=3.0005 | Valid Loss=9.0151 | Train Loss Unweighted=0.5849 | Valid Loss Unweighted=5.2768 | Stale=2 | ETA=863.1m +rank0 [2026-11-14 05:13:29,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:13:30,119] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:13:30,126] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:30,175] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:13:30,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:13:30,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:13:31,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:13:31,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:13:31,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:13:31,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:13:32,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:13:32,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:13:32,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:13:32,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:13:33,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:13:33,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:13:33,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:13:33,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:13:34,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:13:34,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:13:34,438] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:13:34,446] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:35,005] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:35,370] INFO : Plotted number of elements +rank0 [2026-11-14 05:13:36,665] INFO : Plotted jets +rank0 [2026-11-14 05:13:37,628] INFO : Plotted jet ratio +rank0 [2026-11-14 05:14:30,188] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:14:30,188] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:30,536] INFO : Step 41000/500000 rank0 | Train Loss: 1.1779 | Train Loss Unweighted: 0.5557 | Train Loss Components: Classification: 0.0097 | Classification_binary: 0.4637 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0353 | Regression_eta: 0.0026 | Regression_pt: 0.0356 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.1222 | Classification_binary: 0.4637 | Regression_cos_phi: 0.0344 | Regression_energy: 0.1996 | Regression_eta: 0.0165 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0448 | LR: 1.03e-04 | DataLoad Time: 0.0029s | Model Forward Time: 0.4967s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0383 | max_loss: 0.0030 | max_backward: 0.0424 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1195 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2742.4 | uss_MB: 1587.8 | child_rss_MB: 22649.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:15:33,448] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:15:33,448] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:36,355] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:16:36,355] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:22,777] INFO : Step 42000/500000 rank0 | Train Loss: 2.0732 | Train Loss Unweighted: 0.9518 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.7669 | Regression_cos_phi: 0.0018 | Regression_energy: 0.0868 | Regression_eta: 0.0033 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0030 | Classification_binary: 0.7669 | Regression_cos_phi: 0.0162 | Regression_energy: 0.4904 | Regression_eta: 0.0207 | Regression_pt: 0.7562 | Regression_sin_phi: 0.0199 | LR: 1.07e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5335s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0389 | max_loss: 0.0030 | max_backward: 0.0424 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.0 | uss_MB: 1588.3 | child_rss_MB: 22640.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:17:22,779] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:17:22,850] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:17:39,720] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:17:39,720] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:43,110] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:18:43,110] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:15,807] INFO : Step 43000/500000 rank0 | Train Loss: 1.8252 | Train Loss Unweighted: 0.6663 | Train Loss Components: Classification: 0.0037 | Classification_binary: 0.4811 | Regression_cos_phi: 0.0072 | Regression_energy: 0.0766 | Regression_eta: 0.0114 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.4811 | Regression_cos_phi: 0.0662 | Regression_energy: 0.4329 | Regression_eta: 0.0716 | Regression_pt: 0.6208 | Regression_sin_phi: 0.1067 | LR: 1.11e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5149s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1324 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2746.5 | uss_MB: 1591.9 | child_rss_MB: 22663.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:19:46,533] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:19:46,533] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:49,560] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:20:49,560] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:08,370] INFO : Step 44000/500000 rank0 | Train Loss: 0.7894 | Train Loss Unweighted: 0.2774 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0351 | Regression_eta: 0.0059 | Regression_pt: 0.0355 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0076 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0381 | Regression_energy: 0.1981 | Regression_eta: 0.0372 | Regression_pt: 0.2961 | Regression_sin_phi: 0.0181 | LR: 1.15e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5359s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0035 | max_backward: 0.0419 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1221 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2746.8 | uss_MB: 1592.2 | child_rss_MB: 22660.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:21:08,372] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:21:08,441] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:21:53,065] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:21:53,065] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:56,315] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:22:56,315] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:23:01,231] INFO : Step 45000/500000 rank0 | Train Loss: 1.7121 | Train Loss Unweighted: 0.9486 | Train Loss Components: Classification: 0.0007 | Classification_binary: 0.8219 | Regression_cos_phi: 0.0035 | Regression_energy: 0.0612 | Regression_eta: 0.0043 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0086 | Classification_binary: 0.8219 | Regression_cos_phi: 0.0324 | Regression_energy: 0.3459 | Regression_eta: 0.0271 | Regression_pt: 0.4586 | Regression_sin_phi: 0.0176 | LR: 1.19e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5407s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0032 | max_backward: 0.0438 | max_optimizer_step: 0.0715 | max_tensorboard_logging: 0.1388 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2746.8 | uss_MB: 1592.2 | child_rss_MB: 22661.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:23:59,561] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:23:59,561] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:53,279] INFO : Step 46000/500000 rank0 | Train Loss: 1.2403 | Train Loss Unweighted: 0.7231 | Train Loss Components: Classification: 0.0012 | Classification_binary: 0.6401 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0332 | Regression_eta: 0.0054 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0149 | Classification_binary: 0.6401 | Regression_cos_phi: 0.0242 | Regression_energy: 0.1879 | Regression_eta: 0.0337 | Regression_pt: 0.3246 | Regression_sin_phi: 0.0150 | LR: 1.23e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5446s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0032 | max_backward: 0.0407 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1384 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.0 | uss_MB: 1588.4 | child_rss_MB: 22654.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:24:53,281] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 05:24:53,351] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:25:02,666] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 05:25:02,666] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:05,027] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 05:26:05,028] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:44,909] INFO : Step 47000/500000 rank0 | Train Loss: 0.7379 | Train Loss Unweighted: 0.2431 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0311 | Regression_eta: 0.0035 | Regression_pt: 0.0310 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0055 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0244 | Regression_energy: 0.1757 | Regression_eta: 0.0218 | Regression_pt: 0.2586 | Regression_sin_phi: 0.0870 | LR: 1.27e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5003s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0398 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1200 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2745.8 | uss_MB: 1591.4 | child_rss_MB: 22653.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:27:08,046] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 05:27:08,047] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:11,109] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 05:28:11,109] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:37,102] INFO : Step 48000/500000 rank0 | Train Loss: 0.5182 | Train Loss Unweighted: 0.1492 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.0898 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0241 | Regression_eta: 0.0038 | Regression_pt: 0.0254 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0038 | Classification_binary: 0.0898 | Regression_cos_phi: 0.0286 | Regression_energy: 0.1363 | Regression_eta: 0.0236 | Regression_pt: 0.2118 | Regression_sin_phi: 0.0242 | LR: 1.31e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5520s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0028 | max_backward: 0.0411 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1697 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2745.8 | uss_MB: 1591.4 | child_rss_MB: 22664.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:28:37,104] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 05:28:37,186] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:29:14,527] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:29:14,527] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:17,441] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:30:17,442] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:29,774] INFO : Step 49000/500000 rank0 | Train Loss: 0.9632 | Train Loss Unweighted: 0.3099 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0103 | Regression_energy: 0.0392 | Regression_eta: 0.0090 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0010 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0944 | Regression_energy: 0.2212 | Regression_eta: 0.0566 | Regression_pt: 0.3206 | Regression_sin_phi: 0.0636 | LR: 1.36e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5212s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0032 | max_backward: 0.0404 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2745.8 | uss_MB: 1591.5 | child_rss_MB: 22657.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:31:20,840] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:31:20,840] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:22,195] INFO : Step 50000/500000 rank0 | Train Loss: 1.6111 | Train Loss Unweighted: 0.7154 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.5680 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0693 | Regression_eta: 0.0051 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.5904 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1403 | Regression_energy: 5.6497 | Regression_eta: 6.2831 | Regression_pt: 8.3480 | Regression_sin_phi: 8.9384 | Task Weighted Losses: Classification: 0.0177 | Classification_binary: 0.5680 | Regression_cos_phi: 0.0246 | Regression_energy: 0.3915 | Regression_eta: 0.0323 | Regression_pt: 0.5419 | Regression_sin_phi: 0.0350 | LR: 1.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5255s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0028 | max_backward: 0.0422 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1307 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2742.6 | uss_MB: 1588.2 | child_rss_MB: 22654.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:32:22,197] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:32:22,274] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:32:22,275] INFO : Running validation on rank0 +rank0 [2026-11-14 05:32:22,275] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:32:22,275] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:32:22,394] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:32:22,501] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:32:22,604] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:32:22,672] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:32:22,737] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:32:22,804] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:32:22,874] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:32:22,942] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:32:23,054] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:32:23,159] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:32:23,263] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:32:23,331] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:32:23,400] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:32:23,469] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:32:23,538] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:32:23,606] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:32:23,717] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:32:23,724] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:32:23,726] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:32:23,727] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:32:23,730] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:32:23,732] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:32:23,733] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:32:23,735] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:32:23,736] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:32:23,739] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:32:23,741] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:32:23,742] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:32:23,743] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:32:23,745] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:32:23,747] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:32:23,749] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:32:25,189] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.6111 | Valid Loss=10.3299 | Train Loss Unweighted=0.7154 | Valid Loss Unweighted=6.1870 | Stale=3 | ETA=845.8m +rank0 [2026-11-14 05:32:25,190] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:32:25,599] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:32:25,613] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:25,660] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:32:26,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:32:26,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:32:26,659] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:32:26,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:32:27,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:32:27,385] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:32:27,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:32:27,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:32:28,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:32:28,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:32:28,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:32:28,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:32:29,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:32:29,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:32:29,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:32:29,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:32:29,860] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:32:29,868] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:30,404] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2mm_seed12345_20261114_035750_900597/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:30,771] INFO : Plotted number of elements +rank0 [2026-11-14 05:32:32,056] INFO : Plotted jets +rank0 [2026-11-14 05:32:33,020] INFO : Plotted jet ratio +rank0 [2026-11-14 05:32:33,464] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:32:33,464] INFO : Training completed. Total time on device 0: 94.113min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..126c33a87a751e654e82f4acf18aba80ec75dace --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10be0fddfcd992dcd52c7ee23c132c7e5bee67524323c751aca259ac22dd1a4e +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-50000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..cf316725066c063419ab0b08e6bddb41048080e8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5bd39b0bac7a7a2d9ffe02487040281c823b48c9fb518b02847cac22236156d +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..f3cc8b0c3db89b6f8809373987a1683fb1f5cea5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 1.4506076574325562, "Classification": 0.012340160086750984, "Regression_pt": 0.1966983526945114, "Regression_eta": 0.0054677571170032024, "Regression_sin_phi": 0.005530412308871746, "Regression_cos_phi": 0.004872391931712627, "Regression_energy": 0.19303391873836517, "Total": 4.506505012512207, "Total_unweighted": 1.8685506503097713}, "train_group": {}, "valid": {"Classification_binary": 2.4216718673706055, "Classification": 0.014680979773402214, "Regression_pt": 0.2317015826702118, "Regression_eta": 0.016943668946623802, "Regression_sin_phi": 0.020651666447520256, "Regression_cos_phi": 0.01550891064107418, "Regression_energy": 0.2369665652513504, "Total": 6.358796119689941, "Total_unweighted": 2.958125241100788, "metrics/particle/class_1/efficiency": 0.8855835199356079, "metrics/particle/class_1/pid_accuracy": 0.9741601943969727, "metrics/particle/class_2/efficiency": 0.14814814925193787, "metrics/particle/class_2/pid_accuracy": 0.75, "metrics/particle/class_3/efficiency": 0.5555555820465088, "metrics/particle/class_3/pid_accuracy": 0.9927272796630859, "metrics/particle/class_4/efficiency": 0.8389830589294434, "metrics/particle/class_4/pid_accuracy": 0.8888888955116272, "metrics/particle/class_5/efficiency": 0.8941176533699036, "metrics/particle/class_5/pid_accuracy": 0.9736841917037964, "metrics/particle/count/bias_mean": -0.2199999988079071, "metrics/particle/count/mae": 0.6200000047683716, "metrics/particle/count/prediction_mean": 4.427999973297119, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.12443163990974426, "metrics/particle/event/energy_response_mean": 1.0285000801086426, "metrics/particle/event/met_abs_error": 3.047212839126587, "metrics/particle/event/scalar_pt_relative_abs_error": 0.11860701441764832, "metrics/particle/event/scalar_pt_response_mean": 1.0221024751663208, "metrics/particle/event/vector_pt_closure": 0.12138116359710693, "metrics/particle/matched/delta_eta_abs_mean": 0.02755783498287201, "metrics/particle/matched/delta_phi_abs_mean": 0.02337614819407463, "metrics/particle/matched/delta_r_mean": 0.040078502148389816, "metrics/particle/matched/energy_relative_abs_mean": 0.09877856820821762, "metrics/particle/matched/pid_accuracy": 0.9690844416618347, "metrics/particle/matched/pt_relative_abs_mean": 0.09631357342004776, "metrics/particle/matching/duplicate_fraction": 0.03071364015340805, "metrics/particle/matching/efficiency": 0.7237521409988403, "metrics/particle/matching/f1": 0.7412956953048706, "metrics/particle/matching/purity": 0.7597109079360962, "metrics/particle/matching/target_coverage_dr0p05": 0.5275387167930603, "metrics/particle/matching/target_coverage_dr0p10": 0.76161789894104, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6721170544624329, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8132529854774475}, "step_train_time": 0.5302133560180664, "step_valid_time": 2.6728383130393922, "step_total_time": 1106.6108589172363, "elapsed_training_loop_seconds": 1106.6108589172363, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0194956149882017, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11470087978171561, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 399, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 391, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.925754060324826, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9300699300699301, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9279069767441861, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06993006993006994, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9071925754060325, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9114219114219114, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9093023255813953, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.925754060324826, "step_test_time": 5.026226726011373, "step_plot_time": 3.639194782997947, "step_validation_cycle_time": 11.845038464991376} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..7882d578d9da144061e5ae0d145e5872b4026519 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 1.2125213146209717, "Classification": 0.005603093653917313, "Regression_pt": 0.09564796090126038, "Regression_eta": 0.00455683097243309, "Regression_sin_phi": 0.008745028637349606, "Regression_cos_phi": 0.003650224767625332, "Regression_energy": 0.08980894088745117, "Total": 2.747387409210205, "Total_unweighted": 1.4205333944410086}, "train_group": {}, "valid": {"Classification_binary": 2.2919394969940186, "Classification": 0.012763893231749535, "Regression_pt": 0.20242252945899963, "Regression_eta": 0.01131456345319748, "Regression_sin_phi": 0.018864238634705544, "Regression_cos_phi": 0.01254542637616396, "Regression_energy": 0.2110876739025116, "Total": 5.729743003845215, "Total_unweighted": 2.7609378220513463, "metrics/particle/class_1/efficiency": 0.9084668159484863, "metrics/particle/class_1/pid_accuracy": 0.9748110771179199, "metrics/particle/class_2/efficiency": 0.03703703731298447, "metrics/particle/class_2/pid_accuracy": 1.0, "metrics/particle/class_3/efficiency": 0.6080808043479919, "metrics/particle/class_3/pid_accuracy": 0.9867109656333923, "metrics/particle/class_4/efficiency": 0.8813559412956238, "metrics/particle/class_4/pid_accuracy": 0.932692289352417, "metrics/particle/class_5/efficiency": 0.9647058844566345, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.09200000017881393, "metrics/particle/count/mae": 0.5559999942779541, "metrics/particle/count/prediction_mean": 4.556000232696533, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.09527254849672318, "metrics/particle/event/energy_response_mean": 0.998650312423706, "metrics/particle/event/met_abs_error": 2.5159075260162354, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09634938836097717, "metrics/particle/event/scalar_pt_response_mean": 1.0066946744918823, "metrics/particle/event/vector_pt_closure": 0.10282916575670242, "metrics/particle/matched/delta_eta_abs_mean": 0.02456819824874401, "metrics/particle/matched/delta_phi_abs_mean": 0.019670426845550537, "metrics/particle/matched/delta_r_mean": 0.0351518951356411, "metrics/particle/matched/energy_relative_abs_mean": 0.08314073085784912, "metrics/particle/matched/pid_accuracy": 0.9762712121009827, "metrics/particle/matched/pt_relative_abs_mean": 0.08343560993671417, "metrics/particle/matching/duplicate_fraction": 0.021949078887701035, "metrics/particle/matching/efficiency": 0.76161789894104, "metrics/particle/matching/f1": 0.7692307829856873, "metrics/particle/matching/purity": 0.7769973874092102, "metrics/particle/matching/target_coverage_dr0p05": 0.6032702326774597, "metrics/particle/matching/target_coverage_dr0p10": 0.7908778190612793, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7323579788208008, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8433734774589539}, "step_train_time": 0.539736270904541, "step_valid_time": 2.88418073405046, "step_total_time": 2215.1060054302216, "elapsed_training_loop_seconds": 2215.1060054302216, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9906141192259125, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08089891741769728, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 433, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 402, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 398, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9327146171693735, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9284064665127021, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9305555555555556, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07159353348729793, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9234338747099768, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9191685912240185, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9212962962962963, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9327146171693735, "step_test_time": 4.744424440024886, "step_plot_time": 3.1672629070235416, "step_validation_cycle_time": 11.345781836018432} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..2fe3f26c1f3a4ea0a5ff153c779960be133be4b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 1.45589280128479, "Classification": 0.0015472101513296366, "Regression_pt": 0.11078755557537079, "Regression_eta": 0.004135366063565016, "Regression_sin_phi": 0.0030875985976308584, "Regression_cos_phi": 0.00213281880132854, "Regression_energy": 0.10862059891223907, "Total": 3.1105101108551025, "Total_unweighted": 1.686203949386254}, "train_group": {}, "valid": {"Classification_binary": 2.5835952758789062, "Classification": 0.013430079445242882, "Regression_pt": 0.19422608613967896, "Regression_eta": 0.01133911032229662, "Regression_sin_phi": 0.018270576372742653, "Regression_cos_phi": 0.012437018565833569, "Regression_energy": 0.19570475816726685, "Total": 5.8660430908203125, "Total_unweighted": 3.0290029048919678, "metrics/particle/class_1/efficiency": 0.9221968054771423, "metrics/particle/class_1/pid_accuracy": 0.9801489114761353, "metrics/particle/class_2/efficiency": 0.2222222238779068, "metrics/particle/class_2/pid_accuracy": 0.5, "metrics/particle/class_3/efficiency": 0.581818163394928, "metrics/particle/class_3/pid_accuracy": 0.9965277910232544, "metrics/particle/class_4/efficiency": 0.8983050584793091, "metrics/particle/class_4/pid_accuracy": 0.9150943160057068, "metrics/particle/class_5/efficiency": 0.9529411792755127, "metrics/particle/class_5/pid_accuracy": 0.9753086566925049, "metrics/particle/count/bias_mean": -0.25600001215934753, "metrics/particle/count/mae": 0.5519999861717224, "metrics/particle/count/prediction_mean": 4.392000198364258, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.09374293684959412, "metrics/particle/event/energy_response_mean": 1.0282195806503296, "metrics/particle/event/met_abs_error": 2.7865755558013916, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0976962074637413, "metrics/particle/event/scalar_pt_response_mean": 1.0363529920578003, "metrics/particle/event/vector_pt_closure": 0.11132141202688217, "metrics/particle/matched/delta_eta_abs_mean": 0.023194940760731697, "metrics/particle/matched/delta_phi_abs_mean": 0.019002074375748634, "metrics/particle/matched/delta_r_mean": 0.03336697071790695, "metrics/particle/matched/energy_relative_abs_mean": 0.09643062204122543, "metrics/particle/matched/pid_accuracy": 0.9739819169044495, "metrics/particle/matched/pt_relative_abs_mean": 0.09598510712385178, "metrics/particle/matching/duplicate_fraction": 0.0173041895031929, "metrics/particle/matching/efficiency": 0.760757327079773, "metrics/particle/matching/f1": 0.7823008894920349, "metrics/particle/matching/purity": 0.8051002025604248, "metrics/particle/matching/target_coverage_dr0p05": 0.6067125797271729, "metrics/particle/matching/target_coverage_dr0p10": 0.7848536968231201, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7082616090774536, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8339070677757263}, "step_train_time": 0.5196890830993652, "step_valid_time": 2.888890778995119, "step_total_time": 3332.3847036361694, "elapsed_training_loop_seconds": 3332.3847036361694, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0272320030348892, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07877739302335696, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 396, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9373549883990719, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9308755760368663, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9341040462427745, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06912442396313365, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9187935034802784, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9124423963133641, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.915606936416185, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9373549883990719, "step_test_time": 4.720381042046938, "step_plot_time": 3.1865666029625572, "step_validation_cycle_time": 11.253912369022146} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..85c2734d3beacb6644bb039d18d203015246fe6e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 0.7564377784729004, "Classification": 0.019669272005558014, "Regression_pt": 0.04393680766224861, "Regression_eta": 0.004624517634510994, "Regression_sin_phi": 0.0027640920598059893, "Regression_cos_phi": 0.0020693251863121986, "Regression_energy": 0.044117171317338943, "Total": 1.697989583015442, "Total_unweighted": 0.8736189643386751}, "train_group": {}, "valid": {"Classification_binary": 4.372631549835205, "Classification": 0.02099604159593582, "Regression_pt": 0.24249547719955444, "Regression_eta": 0.011772731319069862, "Regression_sin_phi": 0.016948828473687172, "Regression_cos_phi": 0.01286063902080059, "Regression_energy": 0.24002309143543243, "Total": 8.407293319702148, "Total_unweighted": 4.917728358879685, "metrics/particle/class_1/efficiency": 0.9221968054771423, "metrics/particle/class_1/pid_accuracy": 0.9826302528381348, "metrics/particle/class_2/efficiency": 0.07407407462596893, "metrics/particle/class_2/pid_accuracy": 0.5, "metrics/particle/class_3/efficiency": 0.581818163394928, "metrics/particle/class_3/pid_accuracy": 0.9965277910232544, "metrics/particle/class_4/efficiency": 0.8898305296897888, "metrics/particle/class_4/pid_accuracy": 0.9523809552192688, "metrics/particle/class_5/efficiency": 0.9529411792755127, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.03200000151991844, "metrics/particle/count/mae": 0.5920000076293945, "metrics/particle/count/prediction_mean": 4.616000175476074, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.0823257565498352, "metrics/particle/event/energy_response_mean": 0.9912367463111877, "metrics/particle/event/met_abs_error": 2.37943696975708, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08287911862134933, "metrics/particle/event/scalar_pt_response_mean": 1.0011703968048096, "metrics/particle/event/vector_pt_closure": 0.09980937093496323, "metrics/particle/matched/delta_eta_abs_mean": 0.02354361303150654, "metrics/particle/matched/delta_phi_abs_mean": 0.018633047118782997, "metrics/particle/matched/delta_r_mean": 0.03329552710056305, "metrics/particle/matched/energy_relative_abs_mean": 0.08407715708017349, "metrics/particle/matched/pid_accuracy": 0.9840728044509888, "metrics/particle/matched/pt_relative_abs_mean": 0.08376801013946533, "metrics/particle/matching/duplicate_fraction": 0.02253032848238945, "metrics/particle/matching/efficiency": 0.7564544081687927, "metrics/particle/matching/f1": 0.7590673565864563, "metrics/particle/matching/purity": 0.7616984248161316, "metrics/particle/matching/target_coverage_dr0p05": 0.608433723449707, "metrics/particle/matching/target_coverage_dr0p10": 0.781411349773407, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7271944880485535, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.849397599697113}, "step_train_time": 0.557781457901001, "step_valid_time": 2.8989219589857385, "step_total_time": 4447.419093132019, "elapsed_training_loop_seconds": 4447.419093132019, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011989735063835, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0676327236704618, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 399, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 395, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.925754060324826, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9300699300699301, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9279069767441861, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06993006993006994, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9164733178654292, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9207459207459208, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9186046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.925754060324826, "step_test_time": 4.753196115954779, "step_plot_time": 3.2223405269905925, "step_validation_cycle_time": 11.328103318985086} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..8fc6fd6cfac6a485bae6f4e5685c4d4727cb0c0c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 0.6068061590194702, "Classification": 0.000600868312176317, "Regression_pt": 0.0758032351732254, "Regression_eta": 0.006061485968530178, "Regression_sin_phi": 0.005380560178309679, "Regression_cos_phi": 0.0045499037951231, "Regression_energy": 0.06887306272983551, "Total": 1.779898762702942, "Total_unweighted": 0.7680752751766704}, "train_group": {}, "valid": {"Classification_binary": 5.858975410461426, "Classification": 0.03134334832429886, "Regression_pt": 0.2430841028690338, "Regression_eta": 0.01104841660708189, "Regression_sin_phi": 0.017695244401693344, "Regression_cos_phi": 0.013377618044614792, "Regression_energy": 0.2380051612854004, "Total": 10.02177906036377, "Total_unweighted": 6.413529301993549, "metrics/particle/class_1/efficiency": 0.929061770439148, "metrics/particle/class_1/pid_accuracy": 0.9655172228813171, "metrics/particle/class_2/efficiency": 0.14814814925193787, "metrics/particle/class_2/pid_accuracy": 0.25, "metrics/particle/class_3/efficiency": 0.5757575631141663, "metrics/particle/class_3/pid_accuracy": 0.9894737005233765, "metrics/particle/class_4/efficiency": 0.8559321761131287, "metrics/particle/class_4/pid_accuracy": 0.9405940771102905, "metrics/particle/class_5/efficiency": 0.9647058844566345, "metrics/particle/class_5/pid_accuracy": 0.9390243887901306, "metrics/particle/count/bias_mean": -0.06800000369548798, "metrics/particle/count/mae": 0.5559999942779541, "metrics/particle/count/prediction_mean": 4.579999923706055, "metrics/particle/count/target_mean": 4.6479997634887695, "metrics/particle/event/energy_relative_abs_error": 0.08500650525093079, "metrics/particle/event/energy_response_mean": 1.0233088731765747, "metrics/particle/event/met_abs_error": 2.2252447605133057, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08190113306045532, "metrics/particle/event/scalar_pt_response_mean": 1.0206092596054077, "metrics/particle/event/vector_pt_closure": 0.09215566515922546, "metrics/particle/matched/delta_eta_abs_mean": 0.02304806374013424, "metrics/particle/matched/delta_phi_abs_mean": 0.022207334637641907, "metrics/particle/matched/delta_r_mean": 0.03571169078350067, "metrics/particle/matched/energy_relative_abs_mean": 0.07483698427677155, "metrics/particle/matched/pid_accuracy": 0.9646924734115601, "metrics/particle/matched/pt_relative_abs_mean": 0.07171940058469772, "metrics/particle/matching/duplicate_fraction": 0.027074236422777176, "metrics/particle/matching/efficiency": 0.7555937767028809, "metrics/particle/matching/f1": 0.7611616849899292, "metrics/particle/matching/purity": 0.7668122053146362, "metrics/particle/matching/target_coverage_dr0p05": 0.6067125797271729, "metrics/particle/matching/target_coverage_dr0p10": 0.783993124961853, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7280550599098206, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8390705585479736}, "step_train_time": 0.6059529781341553, "step_valid_time": 2.869501408014912, "step_total_time": 5563.159559011459, "elapsed_training_loop_seconds": 5563.159559011459, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0166408740555124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.056193277302343136, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 429, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 399, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 389, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.925754060324826, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9300699300699301, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9279069767441861, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06993006993006994, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9025522041763341, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9067599067599068, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9046511627906977, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.925754060324826, "step_test_time": 4.801363621023484, "step_plot_time": 3.198726880014874, "step_validation_cycle_time": 11.306639015034307} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..655350ddff58e85703ed5b8d416f9ace860be398 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_z_tautau_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..f3445ac3cb27bc36e0e588127e9faf4865b735d1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e03755324f94987db65a3ac30251d21a7570f501d600fa9a6f8e64c20d8bd46 +size 4404 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..784a4de237d594ce8b0e933fbf767b6f39875d9a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..d65c321932c81494cca168fd852d817a5cd6c82b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..eaa889cc66d13c9bc37cf13c51caa1860499fe05 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..d930e733b9d8c9a9b01e247234ca01611eee04d8 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6ff049c6aec652fd7ed786ee9be08c60f814e05c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..3844ff139542b2ebc3ef764257d66d42637c3b44 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6197be962d950f6a9522177e3522c5433053a0a1 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..5089328aad0602058e0fb275f5803fd529a6abf7 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9e9ee12fda31517b07ebfa0f08b4ef551a3d9119 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..672c050b12731076f20e00a25cc495f01483864c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fa11b30a35d55a25da101d7c66ea14f76d5fb671 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..184bcc9b7e7c1bec9753dbca7613065b81d525d5 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/runs/train/events.out.tfevents.1794646765.workergpu154.1460775.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/runs/train/events.out.tfevents.1794646765.workergpu154.1460775.0 new file mode 100644 index 0000000000000000000000000000000000000000..6a44d760d02165c4e152c9b0e59ada9fc1c598fe --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/runs/train/events.out.tfevents.1794646765.workergpu154.1460775.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ad9f92653278d2f2ab9e774eb04ded75cf399db58ee8c20972f56af5ba15a96 +size 1843423 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/runs/valid/events.out.tfevents.1794646765.workergpu154.1460775.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/runs/valid/events.out.tfevents.1794646765.workergpu154.1460775.1 new file mode 100644 index 0000000000000000000000000000000000000000..24b06f3a497c2308eda0fa7dba449ddd42b66654 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/runs/valid/events.out.tfevents.1794646765.workergpu154.1460775.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db987777bc63d3c57d9769d42d5beff58ba4e0e7ed0eb2e2f97ef2118021d1ca +size 2268181 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..14e0c2a901786f44fbc01dec8d0809378d7f11cb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_2p5mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2f07f89dc2f37c844499a7e9dce7cede2d36e35 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_2p5mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/train.log new file mode 100644 index 0000000000000000000000000000000000000000..79871e90f4f91a210fdab66168e0329e8a0f319c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/train.log @@ -0,0 +1,999 @@ +rank0 [2026-11-14 03:59:21,957] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-14 03:59:21,957] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 03:59:22,921] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 03:59:22,921] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 03:59:22,922] INFO : Instantiating model +rank0 [2026-11-14 03:59:22,953] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 03:59:22,953] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 03:59:22,955] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 03:59:22,955] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 03:59:22,955] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 03:59:22,955] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 03:59:22,955] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,958] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 03:59:22,958] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,961] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 03:59:22,961] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,964] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 03:59:22,964] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,966] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 03:59:22,966] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,969] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 03:59:22,969] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,972] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 03:59:22,972] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 03:59:22,972] INFO : Initializing output DNNs +rank0 [2026-11-14 03:59:22,979] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,982] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,990] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-14 03:59:22,990] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 03:59:22,990] INFO : jagged_attention enabled=True +rank0 [2026-11-14 03:59:22,990] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 03:59:22,990] INFO : task_queries enabled=True +rank0 [2026-11-14 03:59:22,990] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 03:59:22,990] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 03:59:22,990] INFO : classification_query parameters: 256 +rank0 [2026-11-14 03:59:22,990] INFO : regression_query parameters: 256 +rank0 [2026-11-14 03:59:22,990] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 03:59:22,990] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 03:59:22,991] INFO : output_mode=elementwise +rank0 [2026-11-14 03:59:22,991] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 03:59:22,991] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 03:59:22,991] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 03:59:22,991] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 03:59:22,991] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 03:59:22,991] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 03:59:22,991] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 03:59:22,991] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 03:59:22,992] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 03:59:22,992] INFO : MLPF __init__ done +rank0 [2026-11-14 03:59:22,993] INFO : Instantiated model +rank0 [2026-11-14 03:59:22,993] INFO : Moving model to device rank=0 +rank0 [2026-11-14 03:59:23,005] INFO : Moved model to device rank=0 +rank0 [2026-11-14 03:59:23,006] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 03:59:23,108] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 03:59:23,108] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 03:59:23,109] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 03:59:23,109] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 03:59:23,109] INFO : Total parameters: 3910930 +rank0 [2026-11-14 03:59:23,112] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 03:59:23,114] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072 +rank0 [2026-11-14 03:59:23,114] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072 +rank0 [2026-11-14 03:59:23,114] INFO : Getting dataloaders +rank0 [2026-11-14 03:59:23,114] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:59:24,978] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,009] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:59:25,052] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:59:25,115] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,144] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,189] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,253] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,294] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,335] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:59:25,366] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,366] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 03:59:25,367] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 03:59:25,367] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 03:59:25,367] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:59:25,375] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,385] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,394] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,403] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,410] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,418] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,425] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,432] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,440] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,447] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,447] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 03:59:25,447] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 03:59:25,500] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:59:25,647] INFO : Got dataloaders +rank0 [2026-11-14 03:59:25,647] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 03:59:25,647] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 03:59:25,648] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 03:59:25,653] INFO : Creating train_iterator +rank0 [2026-11-14 03:59:25,653] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 03:59:25,681] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:59:25,842] INFO : Created train_iterator +rank0 [2026-11-14 03:59:38,304] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.4912013244628906, 'Classification': 0.6057955461740494, 'Regression_pt': 0.8809010863304139, 'Regression_eta': 1.1775247964262963, 'Regression_sin_phi': 0.8549120762944221, 'Regression_cos_phi': 0.8112198185920715, 'Regression_energy': 1.311812883913517}, weights={'Classification_binary': 1.0, 'Classification': 12.365890502929688, 'Regression_pt': 8.504020690917969, 'Regression_eta': 6.361820220947266, 'Regression_sin_phi': 8.762539863586426, 'Regression_cos_phi': 9.234490394592285, 'Regression_energy': 5.710571765899658} +rank0 [2026-11-14 04:00:29,454] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 04:00:29,454] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:17,820] INFO : Step 1000/500000 rank0 | Train Loss: 9.9083 | Train Loss Unweighted: 4.5972 | Train Loss Components: Classification: 0.1353 | Classification_binary: 3.8487 | Regression_cos_phi: 0.0590 | Regression_energy: 0.1505 | Regression_eta: 0.2141 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0317 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 1.6736 | Classification_binary: 3.8487 | Regression_cos_phi: 0.5450 | Regression_energy: 0.8592 | Regression_eta: 1.3623 | Regression_pt: 1.3421 | Regression_sin_phi: 0.2774 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5831s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.2035 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2564.2 | uss_MB: 1405.1 | child_rss_MB: 22467.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:01:31,325] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:01:31,325] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:02:32,392] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:02:32,392] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:06,964] INFO : Step 2000/500000 rank0 | Train Loss: 5.7514 | Train Loss Unweighted: 3.4005 | Train Loss Components: Classification: 0.0472 | Classification_binary: 3.0614 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1272 | Regression_eta: 0.0144 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.5838 | Classification_binary: 3.0614 | Regression_cos_phi: 0.1054 | Regression_energy: 0.7266 | Regression_eta: 0.0915 | Regression_pt: 1.1182 | Regression_sin_phi: 0.0645 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5195s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0030 | max_backward: 0.0392 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1536 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2564.9 | uss_MB: 1405.9 | child_rss_MB: 22460.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:03:06,969] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:03:34,000] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:03:34,001] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:35,529] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:04:35,529] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:56,742] INFO : Step 3000/500000 rank0 | Train Loss: 8.9234 | Train Loss Unweighted: 3.5631 | Train Loss Components: Classification: 0.0387 | Classification_binary: 2.7249 | Regression_cos_phi: 0.0256 | Regression_energy: 0.3390 | Regression_eta: 0.0718 | Regression_pt: 0.3497 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.4789 | Classification_binary: 2.7249 | Regression_cos_phi: 0.2363 | Regression_energy: 1.9359 | Regression_eta: 0.4569 | Regression_pt: 2.9740 | Regression_sin_phi: 0.1166 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5217s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0392 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1338 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2595.5 | uss_MB: 1437.0 | child_rss_MB: 22459.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:05:37,567] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:05:37,567] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:38,909] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:06:38,909] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:46,555] INFO : Step 4000/500000 rank0 | Train Loss: 8.3284 | Train Loss Unweighted: 4.2501 | Train Loss Components: Classification: 0.0321 | Classification_binary: 3.6189 | Regression_cos_phi: 0.0094 | Regression_energy: 0.2708 | Regression_eta: 0.0186 | Regression_pt: 0.2700 | Regression_sin_phi: 0.0301 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.3971 | Classification_binary: 3.6189 | Regression_cos_phi: 0.0869 | Regression_energy: 1.5463 | Regression_eta: 0.1186 | Regression_pt: 2.2965 | Regression_sin_phi: 0.2640 | LR: 2.08e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5251s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0395 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2595.6 | uss_MB: 1437.0 | child_rss_MB: 22459.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:06:46,559] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:07:41,032] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:07:41,032] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:37,065] INFO : Step 5000/500000 rank0 | Train Loss: 5.4880 | Train Loss Unweighted: 3.2653 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.9225 | Regression_cos_phi: 0.0153 | Regression_energy: 0.1422 | Regression_eta: 0.0170 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2227 | Classification_binary: 2.9225 | Regression_cos_phi: 0.1416 | Regression_energy: 0.8121 | Regression_eta: 0.1082 | Regression_pt: 1.1581 | Regression_sin_phi: 0.1228 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5201s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2596.8 | uss_MB: 1438.3 | child_rss_MB: 22460.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:08:43,404] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:08:43,404] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:44,780] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:09:44,780] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:26,850] INFO : Step 6000/500000 rank0 | Train Loss: 11.7303 | Train Loss Unweighted: 4.6051 | Train Loss Components: Classification: 0.0208 | Classification_binary: 3.4784 | Regression_cos_phi: 0.1495 | Regression_energy: 0.4900 | Regression_eta: 0.0768 | Regression_pt: 0.3359 | Regression_sin_phi: 0.0537 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2575 | Classification_binary: 3.4784 | Regression_cos_phi: 1.3801 | Regression_energy: 2.7984 | Regression_eta: 0.4883 | Regression_pt: 2.8567 | Regression_sin_phi: 0.4709 | LR: 2.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5455s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0031 | max_backward: 0.0405 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.1388 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2596.8 | uss_MB: 1438.4 | child_rss_MB: 22459.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:10:26,852] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:10:46,981] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:10:46,982] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:48,904] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:11:48,904] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:12:17,546] INFO : Step 7000/500000 rank0 | Train Loss: 9.0172 | Train Loss Unweighted: 3.9994 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.2114 | Regression_cos_phi: 0.0085 | Regression_energy: 0.3278 | Regression_eta: 0.0179 | Regression_pt: 0.4163 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1636 | Classification_binary: 3.2114 | Regression_cos_phi: 0.0788 | Regression_energy: 1.8718 | Regression_eta: 0.1138 | Regression_pt: 3.5398 | Regression_sin_phi: 0.0379 | LR: 2.26e-05 | DataLoad Time: 0.0049s | Model Forward Time: 0.4844s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1211 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2596.9 | uss_MB: 1438.5 | child_rss_MB: 22459.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:12:51,702] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:12:51,702] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:54,115] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:13:54,116] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:09,210] INFO : Step 8000/500000 rank0 | Train Loss: 5.8763 | Train Loss Unweighted: 2.6389 | Train Loss Components: Classification: 0.0242 | Classification_binary: 2.1372 | Regression_cos_phi: 0.0200 | Regression_energy: 0.1916 | Regression_eta: 0.0477 | Regression_pt: 0.2105 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2997 | Classification_binary: 2.1372 | Regression_cos_phi: 0.1848 | Regression_energy: 1.0944 | Regression_eta: 0.3036 | Regression_pt: 1.7898 | Regression_sin_phi: 0.0668 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5587s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0380 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.1633 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2597.0 | uss_MB: 1438.5 | child_rss_MB: 22458.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:14:09,212] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:14:09,282] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:14:56,698] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:14:56,698] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:57,878] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:15:57,878] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:59,127] INFO : Step 9000/500000 rank0 | Train Loss: 7.9815 | Train Loss Unweighted: 2.9420 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.1436 | Regression_cos_phi: 0.0093 | Regression_energy: 0.3622 | Regression_eta: 0.0160 | Regression_pt: 0.3847 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2783 | Classification_binary: 2.1436 | Regression_cos_phi: 0.0855 | Regression_energy: 2.0683 | Regression_eta: 0.1017 | Regression_pt: 3.2714 | Regression_sin_phi: 0.0326 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5028s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0336 | max_loss: 0.0024 | max_backward: 0.0374 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1527 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2565.1 | uss_MB: 1406.6 | child_rss_MB: 22458.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:17:00,362] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:17:00,363] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:17:49,511] INFO : Step 10000/500000 rank0 | Train Loss: 4.5065 | Train Loss Unweighted: 1.8686 | Train Loss Components: Classification: 0.0123 | Classification_binary: 1.4506 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1930 | Regression_eta: 0.0055 | Regression_pt: 0.1967 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1526 | Classification_binary: 1.4506 | Regression_cos_phi: 0.0450 | Regression_energy: 1.1023 | Regression_eta: 0.0348 | Regression_pt: 1.6727 | Regression_sin_phi: 0.0485 | LR: 2.52e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5296s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1765 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2565.6 | uss_MB: 1407.1 | child_rss_MB: 22459.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:17:49,513] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:17:49,583] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:17:49,584] INFO : Running validation on rank0 +rank0 [2026-11-14 04:17:49,584] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:17:49,584] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:17:49,789] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:17:50,025] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:17:50,095] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:17:50,151] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:17:50,206] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:17:50,253] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:17:50,308] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:17:50,363] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:17:50,411] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:17:50,486] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:17:50,556] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:17:50,611] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:17:50,664] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:17:50,710] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:17:50,762] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:17:50,815] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:17:50,865] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:17:50,869] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:17:50,871] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:17:50,873] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:17:50,875] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:17:50,877] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:17:50,877] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:17:50,879] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:17:50,881] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:17:50,882] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:17:50,883] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:17:50,884] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:17:50,887] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:17:50,889] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:17:50,891] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:17:50,893] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:17:52,334] INFO : VALIDATION | Step=10000/500000 | Train Loss=4.5065 | Valid Loss=6.3588 | Train Loss Unweighted=1.8686 | Valid Loss Unweighted=2.9581 | Stale=0 | ETA=903.8m +rank0 [2026-11-14 04:17:52,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:17:52,939] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:17:52,946] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:53,000] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:17:53,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:17:53,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:17:54,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:17:54,456] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:17:54,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:17:54,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:17:55,171] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:17:55,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:17:55,656] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:17:55,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:17:56,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:17:56,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:17:56,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:17:56,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:17:57,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:17:57,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:17:57,358] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:17:57,366] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:57,898] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:58,796] INFO : Plotted number of elements +rank0 [2026-11-14 04:18:00,060] INFO : Plotted jets +rank0 [2026-11-14 04:18:00,990] INFO : Plotted jet ratio +rank0 [2026-11-14 04:18:13,669] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:18:13,669] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:15,209] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:19:15,209] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:51,337] INFO : Step 11000/500000 rank0 | Train Loss: 3.2896 | Train Loss Unweighted: 1.7410 | Train Loss Components: Classification: 0.0140 | Classification_binary: 1.5015 | Regression_cos_phi: 0.0036 | Regression_energy: 0.1052 | Regression_eta: 0.0061 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1726 | Classification_binary: 1.5015 | Regression_cos_phi: 0.0331 | Regression_energy: 0.6010 | Regression_eta: 0.0389 | Regression_pt: 0.8697 | Regression_sin_phi: 0.0728 | LR: 2.63e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5321s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1339 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2702.2 | uss_MB: 1556.9 | child_rss_MB: 22461.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:20:17,835] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:20:17,835] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:19,692] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:21:19,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:41,699] INFO : Step 12000/500000 rank0 | Train Loss: 3.8566 | Train Loss Unweighted: 2.5132 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.3122 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0827 | Regression_eta: 0.0065 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2393 | Classification_binary: 2.3122 | Regression_cos_phi: 0.0456 | Regression_energy: 0.4720 | Regression_eta: 0.0416 | Regression_pt: 0.6974 | Regression_sin_phi: 0.0485 | LR: 2.75e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5130s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0342 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0640 | max_tensorboard_logging: 0.1472 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2702.2 | uss_MB: 1556.9 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:21:41,701] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:21:41,770] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:22:21,423] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:22:21,423] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:22,964] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:23:22,964] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:31,436] INFO : Step 13000/500000 rank0 | Train Loss: 8.5581 | Train Loss Unweighted: 3.3389 | Train Loss Components: Classification: 0.0064 | Classification_binary: 2.4967 | Regression_cos_phi: 0.0058 | Regression_energy: 0.4022 | Regression_eta: 0.0049 | Regression_pt: 0.4060 | Regression_sin_phi: 0.0169 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0788 | Classification_binary: 2.4967 | Regression_cos_phi: 0.0536 | Regression_energy: 2.2970 | Regression_eta: 0.0312 | Regression_pt: 3.4524 | Regression_sin_phi: 0.1484 | LR: 2.88e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5075s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0641 | max_tensorboard_logging: 0.1490 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2645.1 | uss_MB: 1499.8 | child_rss_MB: 22461.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:24,341] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:24:24,341] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:20,419] INFO : Step 14000/500000 rank0 | Train Loss: 5.4174 | Train Loss Unweighted: 2.1849 | Train Loss Components: Classification: 0.0064 | Classification_binary: 1.6464 | Regression_cos_phi: 0.0059 | Regression_energy: 0.2917 | Regression_eta: 0.0115 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0788 | Classification_binary: 1.6464 | Regression_cos_phi: 0.0546 | Regression_energy: 1.6657 | Regression_eta: 0.0731 | Regression_pt: 1.8145 | Regression_sin_phi: 0.0844 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5188s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0032 | max_backward: 0.0404 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1358 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2645.7 | uss_MB: 1500.4 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:25:20,421] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:25:20,489] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:25:25,905] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:25:25,905] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:26,933] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:26:26,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:09,928] INFO : Step 15000/500000 rank0 | Train Loss: 3.5910 | Train Loss Unweighted: 2.4022 | Train Loss Components: Classification: 0.0117 | Classification_binary: 2.2218 | Regression_cos_phi: 0.0066 | Regression_energy: 0.0709 | Regression_eta: 0.0085 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1444 | Classification_binary: 2.2218 | Regression_cos_phi: 0.0610 | Regression_energy: 0.4050 | Regression_eta: 0.0540 | Regression_pt: 0.6630 | Regression_sin_phi: 0.0418 | LR: 3.17e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5367s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1669 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2647.4 | uss_MB: 1502.1 | child_rss_MB: 22461.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:27:28,727] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:27:28,727] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:29,721] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:28:29,722] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:58,838] INFO : Step 16000/500000 rank0 | Train Loss: 2.1657 | Train Loss Unweighted: 1.2431 | Train Loss Components: Classification: 0.0016 | Classification_binary: 1.0977 | Regression_cos_phi: 0.0086 | Regression_energy: 0.0589 | Regression_eta: 0.0080 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0198 | Classification_binary: 1.0977 | Regression_cos_phi: 0.0797 | Regression_energy: 0.3364 | Regression_eta: 0.0507 | Regression_pt: 0.5606 | Regression_sin_phi: 0.0207 | LR: 3.33e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5485s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0030 | max_backward: 0.0397 | max_optimizer_step: 0.0646 | max_tensorboard_logging: 0.1539 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2647.5 | uss_MB: 1502.2 | child_rss_MB: 22459.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:28:58,840] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:58,909] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:29:31,237] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:29:31,237] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:32,003] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:30:32,003] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:47,739] INFO : Step 17000/500000 rank0 | Train Loss: 4.3476 | Train Loss Unweighted: 2.5891 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3193 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1155 | Regression_eta: 0.0096 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2244 | Classification_binary: 2.3193 | Regression_cos_phi: 0.0621 | Regression_energy: 0.6593 | Regression_eta: 0.0613 | Regression_pt: 0.9461 | Regression_sin_phi: 0.0750 | LR: 3.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5310s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.1339 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2645.1 | uss_MB: 1499.8 | child_rss_MB: 22459.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:31:34,168] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:31:34,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:35,462] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:32:35,462] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:37,657] INFO : Step 18000/500000 rank0 | Train Loss: 2.5472 | Train Loss Unweighted: 1.6154 | Train Loss Components: Classification: 0.0299 | Classification_binary: 1.4904 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0421 | Regression_eta: 0.0038 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.3699 | Classification_binary: 1.4904 | Regression_cos_phi: 0.0316 | Regression_energy: 0.2402 | Regression_eta: 0.0239 | Regression_pt: 0.3665 | Regression_sin_phi: 0.0246 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5379s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0027 | max_backward: 0.0381 | max_optimizer_step: 0.0646 | max_tensorboard_logging: 0.1627 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2646.2 | uss_MB: 1500.9 | child_rss_MB: 22461.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:32:37,659] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:32:37,729] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:33:37,167] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:33:37,167] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:27,254] INFO : Step 19000/500000 rank0 | Train Loss: 5.9953 | Train Loss Unweighted: 3.6054 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.2414 | Regression_cos_phi: 0.0172 | Regression_energy: 0.1512 | Regression_eta: 0.0091 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2767 | Classification_binary: 3.2414 | Regression_cos_phi: 0.1590 | Regression_energy: 0.8634 | Regression_eta: 0.0581 | Regression_pt: 1.3402 | Regression_sin_phi: 0.0565 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5494s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1568 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2647.7 | uss_MB: 1502.4 | child_rss_MB: 22460.7 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:34:39,051] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:34:39,052] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:40,616] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:35:40,616] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:17,804] INFO : Step 20000/500000 rank0 | Train Loss: 2.7474 | Train Loss Unweighted: 1.4205 | Train Loss Components: Classification: 0.0056 | Classification_binary: 1.2125 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0898 | Regression_eta: 0.0046 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 1.2125 | Regression_cos_phi: 0.0337 | Regression_energy: 0.5129 | Regression_eta: 0.0290 | Regression_pt: 0.8134 | Regression_sin_phi: 0.0766 | LR: 4.07e-05 | DataLoad Time: 0.0063s | Model Forward Time: 0.5333s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0031 | max_backward: 0.0407 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1562 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2648.0 | uss_MB: 1502.7 | child_rss_MB: 22460.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:36:17,806] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:36:17,872] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:36:17,873] INFO : Running validation on rank0 +rank0 [2026-11-14 04:36:17,873] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:36:17,873] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:36:17,989] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:36:18,101] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:36:18,206] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:36:18,275] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:36:18,341] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:36:18,405] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:36:18,474] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:36:18,544] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:36:18,646] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:36:18,752] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:36:18,856] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:36:18,923] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:36:18,989] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:36:19,054] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:36:19,121] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:36:19,189] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:36:19,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:36:19,312] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:36:19,315] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:36:19,317] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:36:19,319] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:36:19,321] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:36:19,322] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:36:19,323] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:36:19,325] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:36:19,326] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:36:19,328] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:36:19,330] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:36:19,332] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:36:19,334] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:36:19,335] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:36:19,338] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:36:20,831] INFO : VALIDATION | Step=20000/500000 | Train Loss=2.7474 | Valid Loss=5.7297 | Train Loss Unweighted=1.4205 | Valid Loss Unweighted=2.7609 | Stale=0 | ETA=886.1m +rank0 [2026-11-14 04:36:20,831] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:36:21,380] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:36:21,387] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:36:21,438] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:36:21,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:36:22,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:36:22,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:36:22,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:36:22,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:36:23,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:36:23,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:36:23,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:36:23,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:36:24,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:36:24,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:36:24,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:36:24,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:36:25,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:36:25,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:36:25,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:36:25,573] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:36:25,580] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:36:26,127] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:36:26,533] INFO : Plotted number of elements +rank0 [2026-11-14 04:36:27,791] INFO : Plotted jets +rank0 [2026-11-14 04:36:28,734] INFO : Plotted jet ratio +rank0 [2026-11-14 04:36:54,747] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:36:54,747] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:56,528] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:37:56,528] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:38:19,589] INFO : Step 21000/500000 rank0 | Train Loss: 3.9015 | Train Loss Unweighted: 2.5558 | Train Loss Components: Classification: 0.0064 | Classification_binary: 2.3400 | Regression_cos_phi: 0.0035 | Regression_energy: 0.1040 | Regression_eta: 0.0053 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 2.3400 | Regression_cos_phi: 0.0324 | Regression_energy: 0.5941 | Regression_eta: 0.0340 | Regression_pt: 0.7973 | Regression_sin_phi: 0.0246 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5364s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0027 | max_backward: 0.0443 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1435 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2675.5 | uss_MB: 1530.2 | child_rss_MB: 22461.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:38:58,646] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:38:58,646] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:00,474] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:40:00,474] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:09,514] INFO : Step 22000/500000 rank0 | Train Loss: 3.0897 | Train Loss Unweighted: 1.8899 | Train Loss Components: Classification: 0.0102 | Classification_binary: 1.7062 | Regression_cos_phi: 0.0015 | Regression_energy: 0.0721 | Regression_eta: 0.0088 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1258 | Classification_binary: 1.7062 | Regression_cos_phi: 0.0142 | Regression_energy: 0.4119 | Regression_eta: 0.0558 | Regression_pt: 0.7336 | Regression_sin_phi: 0.0422 | LR: 4.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4924s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.1501 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2676.3 | uss_MB: 1531.0 | child_rss_MB: 22461.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:40:09,515] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:40:09,583] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:41:02,892] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:41:02,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:00,918] INFO : Step 23000/500000 rank0 | Train Loss: 3.7919 | Train Loss Unweighted: 2.5744 | Train Loss Components: Classification: 0.0105 | Classification_binary: 2.3864 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0807 | Regression_eta: 0.0056 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1303 | Classification_binary: 2.3864 | Regression_cos_phi: 0.0400 | Regression_energy: 0.4611 | Regression_eta: 0.0354 | Regression_pt: 0.7060 | Regression_sin_phi: 0.0327 | LR: 4.73e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0029 | max_backward: 0.0391 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1490 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2678.6 | uss_MB: 1533.3 | child_rss_MB: 22460.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:42:05,371] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:42:05,371] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:05,993] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:43:05,993] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:50,110] INFO : Step 24000/500000 rank0 | Train Loss: 7.1134 | Train Loss Unweighted: 2.5597 | Train Loss Components: Classification: 0.0063 | Classification_binary: 1.8237 | Regression_cos_phi: 0.0067 | Regression_energy: 0.3491 | Regression_eta: 0.0123 | Regression_pt: 0.3505 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 1.8237 | Regression_cos_phi: 0.0620 | Regression_energy: 1.9937 | Regression_eta: 0.0782 | Regression_pt: 2.9808 | Regression_sin_phi: 0.0969 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5703s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0700 | max_tensorboard_logging: 0.1830 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.6 | uss_MB: 1533.3 | child_rss_MB: 22461.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:43:50,112] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:43:50,183] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:44:08,397] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:44:08,397] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:10,609] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:45:10,609] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:41,759] INFO : Step 25000/500000 rank0 | Train Loss: 4.1535 | Train Loss Unweighted: 2.3185 | Train Loss Components: Classification: 0.0081 | Classification_binary: 2.0293 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1287 | Regression_eta: 0.0074 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1005 | Classification_binary: 2.0293 | Regression_cos_phi: 0.0876 | Regression_energy: 0.7348 | Regression_eta: 0.0470 | Regression_pt: 1.0891 | Regression_sin_phi: 0.0651 | LR: 5.22e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5149s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0367 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0700 | max_tensorboard_logging: 0.1206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2675.6 | uss_MB: 1530.3 | child_rss_MB: 22460.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:46:14,015] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:46:14,015] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:16,337] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:47:16,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:33,133] INFO : Step 26000/500000 rank0 | Train Loss: 2.6816 | Train Loss Unweighted: 1.7997 | Train Loss Components: Classification: 0.0025 | Classification_binary: 1.6599 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0595 | Regression_eta: 0.0066 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 1.6599 | Regression_cos_phi: 0.0360 | Regression_energy: 0.3397 | Regression_eta: 0.0420 | Regression_pt: 0.5390 | Regression_sin_phi: 0.0337 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5572s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0031 | max_backward: 0.0395 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1530 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2676.4 | uss_MB: 1531.1 | child_rss_MB: 22462.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:47:33,135] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:47:33,208] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:48:18,897] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:48:18,897] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:20,965] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:49:20,965] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:23,998] INFO : Step 27000/500000 rank0 | Train Loss: 2.7217 | Train Loss Unweighted: 1.6031 | Train Loss Components: Classification: 0.0028 | Classification_binary: 1.4216 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0839 | Regression_eta: 0.0113 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 1.4216 | Regression_cos_phi: 0.0311 | Regression_energy: 0.4792 | Regression_eta: 0.0717 | Regression_pt: 0.6254 | Regression_sin_phi: 0.0582 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5158s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1212 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.3 | uss_MB: 1533.0 | child_rss_MB: 22462.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:50:23,447] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:50:23,447] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:14,554] INFO : Step 28000/500000 rank0 | Train Loss: 2.7285 | Train Loss Unweighted: 1.6226 | Train Loss Components: Classification: 0.0137 | Classification_binary: 1.4558 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0610 | Regression_eta: 0.0153 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1690 | Classification_binary: 1.4558 | Regression_cos_phi: 0.0447 | Regression_energy: 0.3482 | Regression_eta: 0.0975 | Regression_pt: 0.5502 | Regression_sin_phi: 0.0631 | LR: 6.01e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5268s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0783 | max_tensorboard_logging: 0.1239 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.5 | uss_MB: 1533.2 | child_rss_MB: 22461.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:51:14,556] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:51:14,623] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:51:25,568] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:51:25,568] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:27,151] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:52:27,152] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:04,891] INFO : Step 29000/500000 rank0 | Train Loss: 2.8718 | Train Loss Unweighted: 1.5655 | Train Loss Components: Classification: 0.0093 | Classification_binary: 1.3615 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0912 | Regression_eta: 0.0045 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1148 | Classification_binary: 1.3615 | Regression_cos_phi: 0.0242 | Regression_energy: 0.5210 | Regression_eta: 0.0287 | Regression_pt: 0.7577 | Regression_sin_phi: 0.0640 | LR: 6.29e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5237s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0744 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.5 | uss_MB: 1533.2 | child_rss_MB: 22461.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:53:29,550] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:53:29,553] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:31,269] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:54:31,269] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:55,077] INFO : Step 30000/500000 rank0 | Train Loss: 3.1105 | Train Loss Unweighted: 1.6862 | Train Loss Components: Classification: 0.0015 | Classification_binary: 1.4559 | Regression_cos_phi: 0.0021 | Regression_energy: 0.1086 | Regression_eta: 0.0041 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0191 | Classification_binary: 1.4559 | Regression_cos_phi: 0.0197 | Regression_energy: 0.6203 | Regression_eta: 0.0263 | Regression_pt: 0.9421 | Regression_sin_phi: 0.0271 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5188s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0383 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2676.4 | uss_MB: 1531.1 | child_rss_MB: 22462.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:54:55,079] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:54:55,146] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:54:55,147] INFO : Running validation on rank0 +rank0 [2026-11-14 04:54:55,147] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:54:55,147] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:54:55,274] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:54:55,384] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:54:55,488] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:54:55,556] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:54:55,624] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:54:55,689] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:54:55,755] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:54:55,821] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:54:55,923] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:54:56,032] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:54:56,142] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:54:56,209] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:54:56,276] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:54:56,342] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:54:56,408] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:54:56,475] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:54:56,577] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:54:56,583] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:54:56,585] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:54:56,588] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:54:56,590] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:54:56,591] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:54:56,592] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:54:56,594] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:54:56,596] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:54:56,597] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:54:56,598] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:54:56,600] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:54:56,602] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:54:56,604] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:54:56,606] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:54:56,608] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:54:58,040] INFO : VALIDATION | Step=30000/500000 | Train Loss=3.1105 | Valid Loss=5.8660 | Train Loss Unweighted=1.6862 | Valid Loss Unweighted=3.0290 | Stale=1 | ETA=870.1m +rank0 [2026-11-14 04:54:58,040] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:54:58,554] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:54:58,563] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:58,611] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:54:59,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:54:59,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:54:59,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:54:59,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:55:00,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:55:00,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:55:00,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:55:00,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:55:01,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:55:01,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:55:01,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:55:01,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:55:01,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:55:02,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:55:02,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:55:02,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:55:02,759] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:55:02,765] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:55:03,304] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:55:03,716] INFO : Plotted number of elements +rank0 [2026-11-14 04:55:04,989] INFO : Plotted jets +rank0 [2026-11-14 04:55:05,938] INFO : Plotted jet ratio +rank0 [2026-11-14 04:55:44,662] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:55:44,662] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:46,562] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:56:46,563] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:56,907] INFO : Step 31000/500000 rank0 | Train Loss: 2.8743 | Train Loss Unweighted: 1.9270 | Train Loss Components: Classification: 0.0106 | Classification_binary: 1.7826 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0609 | Regression_eta: 0.0048 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1307 | Classification_binary: 1.7826 | Regression_cos_phi: 0.0317 | Regression_energy: 0.3479 | Regression_eta: 0.0308 | Regression_pt: 0.5172 | Regression_sin_phi: 0.0335 | LR: 6.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5363s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0032 | max_backward: 0.0403 | max_optimizer_step: 0.0714 | max_tensorboard_logging: 0.1221 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2793.2 | uss_MB: 1648.0 | child_rss_MB: 22461.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:57:49,032] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:57:49,032] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:47,702] INFO : Step 32000/500000 rank0 | Train Loss: 3.9157 | Train Loss Unweighted: 2.6112 | Train Loss Components: Classification: 0.0032 | Classification_binary: 2.3997 | Regression_cos_phi: 0.0041 | Regression_energy: 0.1018 | Regression_eta: 0.0070 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 2.3997 | Regression_cos_phi: 0.0377 | Regression_energy: 0.5811 | Regression_eta: 0.0446 | Regression_pt: 0.7730 | Regression_sin_phi: 0.0399 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5120s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0396 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1236 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2793.2 | uss_MB: 1648.0 | child_rss_MB: 22461.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:58:47,704] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:58:47,773] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:58:51,515] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:58:51,515] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:53,626] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:59:53,627] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:38,454] INFO : Step 33000/500000 rank0 | Train Loss: 4.4119 | Train Loss Unweighted: 1.7205 | Train Loss Components: Classification: 0.0067 | Classification_binary: 1.2896 | Regression_cos_phi: 0.0075 | Regression_energy: 0.2007 | Regression_eta: 0.0066 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 1.2896 | Regression_cos_phi: 0.0694 | Regression_energy: 1.1459 | Regression_eta: 0.0422 | Regression_pt: 1.7336 | Regression_sin_phi: 0.0480 | LR: 7.51e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5115s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0380 | max_loss: 0.0033 | max_backward: 0.0388 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1315 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2701.6 | uss_MB: 1556.3 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:00:55,744] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:00:55,744] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:57,701] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:01:57,701] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:28,799] INFO : Step 34000/500000 rank0 | Train Loss: 3.3924 | Train Loss Unweighted: 1.5938 | Train Loss Components: Classification: 0.0042 | Classification_binary: 1.3043 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1379 | Regression_eta: 0.0033 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 1.3043 | Regression_cos_phi: 0.0143 | Regression_energy: 0.7873 | Regression_eta: 0.0207 | Regression_pt: 1.1908 | Regression_sin_phi: 0.0228 | LR: 7.83e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5289s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0405 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1318 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2702.1 | uss_MB: 1556.8 | child_rss_MB: 22463.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:02:28,801] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:02:28,871] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:02:59,789] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:02:59,789] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:01,481] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:04:01,481] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:18,771] INFO : Step 35000/500000 rank0 | Train Loss: 1.1370 | Train Loss Unweighted: 0.5912 | Train Loss Components: Classification: 0.0028 | Classification_binary: 0.5060 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0337 | Regression_eta: 0.0059 | Regression_pt: 0.0373 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.5060 | Regression_cos_phi: 0.0250 | Regression_energy: 0.1922 | Regression_eta: 0.0375 | Regression_pt: 0.3171 | Regression_sin_phi: 0.0247 | LR: 8.16e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5029s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0027 | max_backward: 0.0390 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1516 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2704.6 | uss_MB: 1559.3 | child_rss_MB: 22462.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:05:02,463] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:05:02,463] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:05,555] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:06:05,556] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:09,388] INFO : Step 36000/500000 rank0 | Train Loss: 3.3995 | Train Loss Unweighted: 1.9744 | Train Loss Components: Classification: 0.0061 | Classification_binary: 1.7477 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1038 | Regression_eta: 0.0066 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 1.7477 | Regression_cos_phi: 0.0418 | Regression_energy: 0.5927 | Regression_eta: 0.0418 | Regression_pt: 0.8678 | Regression_sin_phi: 0.0324 | LR: 8.50e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5484s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0382 | max_optimizer_step: 0.0645 | max_tensorboard_logging: 0.1623 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.6 | uss_MB: 1559.4 | child_rss_MB: 22461.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:06:09,390] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:06:09,459] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:07:05,606] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:07:05,606] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:57,921] INFO : Step 37000/500000 rank0 | Train Loss: 2.4504 | Train Loss Unweighted: 1.5685 | Train Loss Components: Classification: 0.0014 | Classification_binary: 1.4253 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0647 | Regression_eta: 0.0100 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0168 | Classification_binary: 1.4253 | Regression_cos_phi: 0.0472 | Regression_energy: 0.3696 | Regression_eta: 0.0639 | Regression_pt: 0.5050 | Regression_sin_phi: 0.0225 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5338s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0030 | max_backward: 0.0410 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2704.7 | uss_MB: 1559.4 | child_rss_MB: 22461.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:08:08,023] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:08:08,023] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:10,229] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:09:10,229] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:48,688] INFO : Step 38000/500000 rank0 | Train Loss: 2.0249 | Train Loss Unweighted: 1.0969 | Train Loss Components: Classification: 0.0077 | Classification_binary: 0.9528 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0630 | Regression_eta: 0.0050 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0955 | Classification_binary: 0.9528 | Regression_cos_phi: 0.0255 | Regression_energy: 0.3597 | Regression_eta: 0.0318 | Regression_pt: 0.5129 | Regression_sin_phi: 0.0467 | LR: 9.21e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6119s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0028 | max_backward: 0.0387 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.2231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2702.4 | uss_MB: 1557.2 | child_rss_MB: 22462.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:09:48,690] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:09:48,761] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:10:12,578] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:10:12,579] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:14,280] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:11:14,280] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:39,105] INFO : Step 39000/500000 rank0 | Train Loss: 2.0043 | Train Loss Unweighted: 1.1855 | Train Loss Components: Classification: 0.0013 | Classification_binary: 1.0551 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0547 | Regression_eta: 0.0082 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0156 | Classification_binary: 1.0551 | Regression_cos_phi: 0.0493 | Regression_energy: 0.3125 | Regression_eta: 0.0520 | Regression_pt: 0.4751 | Regression_sin_phi: 0.0446 | LR: 9.57e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5248s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0030 | max_backward: 0.0390 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1366 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2704.9 | uss_MB: 1559.7 | child_rss_MB: 22461.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:12:16,164] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:12:16,165] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:18,719] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:13:18,719] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:30,102] INFO : Step 40000/500000 rank0 | Train Loss: 1.6980 | Train Loss Unweighted: 0.8736 | Train Loss Components: Classification: 0.0197 | Classification_binary: 0.7564 | Regression_cos_phi: 0.0021 | Regression_energy: 0.0441 | Regression_eta: 0.0046 | Regression_pt: 0.0439 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2432 | Classification_binary: 0.7564 | Regression_cos_phi: 0.0191 | Regression_energy: 0.2519 | Regression_eta: 0.0294 | Regression_pt: 0.3736 | Regression_sin_phi: 0.0242 | LR: 9.94e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5569s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0030 | max_backward: 0.0396 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1550 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2705.1 | uss_MB: 1559.8 | child_rss_MB: 22460.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:13:30,104] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:13:30,170] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:13:30,171] INFO : Running validation on rank0 +rank0 [2026-11-14 05:13:30,171] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:13:30,171] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:13:30,290] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:13:30,396] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:13:30,494] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:13:30,563] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:13:30,630] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:13:30,695] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:13:30,761] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:13:30,829] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:13:30,942] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:13:31,058] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:13:31,172] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:13:31,242] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:13:31,311] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:13:31,379] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:13:31,443] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:13:31,511] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:13:31,609] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:13:31,615] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:13:31,617] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:13:31,619] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:13:31,620] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:13:31,623] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:13:31,624] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:13:31,626] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:13:31,628] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:13:31,629] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:13:31,630] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:13:31,632] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:13:31,634] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:13:31,636] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:13:31,638] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:13:31,640] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:13:33,074] INFO : VALIDATION | Step=40000/500000 | Train Loss=1.6980 | Valid Loss=8.4073 | Train Loss Unweighted=0.8736 | Valid Loss Unweighted=4.9177 | Stale=2 | ETA=852.4m +rank0 [2026-11-14 05:13:33,075] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:13:33,546] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:13:33,554] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:33,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:13:34,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:13:34,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:13:34,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:13:34,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:13:35,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:13:35,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:13:35,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:13:35,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:13:36,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:13:36,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:13:36,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:13:36,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:13:37,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:13:37,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:13:37,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:13:37,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:13:37,826] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:13:37,832] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:38,384] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:38,788] INFO : Plotted number of elements +rank0 [2026-11-14 05:13:40,079] INFO : Plotted jets +rank0 [2026-11-14 05:13:41,043] INFO : Plotted jet ratio +rank0 [2026-11-14 05:14:32,812] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:14:32,812] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:32,277] INFO : Step 41000/500000 rank0 | Train Loss: 1.2421 | Train Loss Unweighted: 0.5587 | Train Loss Components: Classification: 0.0010 | Classification_binary: 0.4492 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0468 | Regression_eta: 0.0066 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0128 | Classification_binary: 0.4492 | Regression_cos_phi: 0.0285 | Regression_energy: 0.2674 | Regression_eta: 0.0422 | Regression_pt: 0.4020 | Regression_sin_phi: 0.0400 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5483s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0031 | max_backward: 0.0397 | max_optimizer_step: 0.0712 | max_tensorboard_logging: 0.1393 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.2 | uss_MB: 1581.0 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:15:35,097] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:15:35,097] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:36,613] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:16:36,613] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:22,268] INFO : Step 42000/500000 rank0 | Train Loss: 1.9269 | Train Loss Unweighted: 1.0489 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.9050 | Regression_cos_phi: 0.0024 | Regression_energy: 0.0645 | Regression_eta: 0.0118 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0052 | Classification_binary: 0.9050 | Regression_cos_phi: 0.0226 | Regression_energy: 0.3681 | Regression_eta: 0.0754 | Regression_pt: 0.5343 | Regression_sin_phi: 0.0164 | LR: 1.07e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5923s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0392 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1958 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.7 | uss_MB: 1581.5 | child_rss_MB: 22462.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:17:22,270] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:17:22,341] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:17:38,753] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:17:38,753] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:41,099] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:18:41,099] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:13,310] INFO : Step 43000/500000 rank0 | Train Loss: 1.7223 | Train Loss Unweighted: 0.7572 | Train Loss Components: Classification: 0.0042 | Classification_binary: 0.6069 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0579 | Regression_eta: 0.0114 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0172 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.6069 | Regression_cos_phi: 0.0396 | Regression_energy: 0.3304 | Regression_eta: 0.0725 | Regression_pt: 0.4699 | Regression_sin_phi: 0.1509 | LR: 1.11e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5376s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1363 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.5 | uss_MB: 1585.3 | child_rss_MB: 22462.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:19:43,456] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:19:43,456] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:45,108] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:20:45,108] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:03,550] INFO : Step 44000/500000 rank0 | Train Loss: 0.8604 | Train Loss Unweighted: 0.4138 | Train Loss Components: Classification: 0.0010 | Classification_binary: 0.3417 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0325 | Regression_eta: 0.0044 | Regression_pt: 0.0299 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0121 | Classification_binary: 0.3417 | Regression_cos_phi: 0.0189 | Regression_energy: 0.1854 | Regression_eta: 0.0279 | Regression_pt: 0.2539 | Regression_sin_phi: 0.0205 | LR: 1.15e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5495s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0638 | max_tensorboard_logging: 0.1603 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2730.9 | uss_MB: 1585.6 | child_rss_MB: 22461.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:21:03,552] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:21:03,623] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:21:47,382] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:21:47,382] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:49,878] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:22:49,878] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:54,751] INFO : Step 45000/500000 rank0 | Train Loss: 2.0273 | Train Loss Unweighted: 0.9366 | Train Loss Components: Classification: 0.0011 | Classification_binary: 0.7614 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0799 | Regression_eta: 0.0043 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0137 | Classification_binary: 0.7614 | Regression_cos_phi: 0.0357 | Regression_energy: 0.4561 | Regression_eta: 0.0275 | Regression_pt: 0.6871 | Regression_sin_phi: 0.0458 | LR: 1.19e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5689s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0366 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1757 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2731.2 | uss_MB: 1586.0 | child_rss_MB: 22462.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:23:52,095] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:23:52,095] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:45,006] INFO : Step 46000/500000 rank0 | Train Loss: 1.3640 | Train Loss Unweighted: 0.8152 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.7303 | Regression_cos_phi: 0.0097 | Regression_energy: 0.0324 | Regression_eta: 0.0038 | Regression_pt: 0.0359 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0065 | Classification_binary: 0.7303 | Regression_cos_phi: 0.0892 | Regression_energy: 0.1848 | Regression_eta: 0.0244 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0235 | LR: 1.23e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5132s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0711 | max_tensorboard_logging: 0.1285 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.8 | uss_MB: 1581.5 | child_rss_MB: 22461.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:24:45,008] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 05:24:45,080] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:24:54,333] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 05:24:54,333] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:56,157] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 05:25:56,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:35,484] INFO : Step 47000/500000 rank0 | Train Loss: 2.2112 | Train Loss Unweighted: 0.6465 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.3918 | Regression_cos_phi: 0.0029 | Regression_energy: 0.1223 | Regression_eta: 0.0041 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0055 | Classification_binary: 0.3918 | Regression_cos_phi: 0.0272 | Regression_energy: 0.6984 | Regression_eta: 0.0262 | Regression_pt: 1.0432 | Regression_sin_phi: 0.0188 | LR: 1.27e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5146s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0395 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1224 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2730.5 | uss_MB: 1585.3 | child_rss_MB: 22462.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:26:58,368] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 05:26:58,368] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:00,375] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 05:28:00,375] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:25,970] INFO : Step 48000/500000 rank0 | Train Loss: 2.6417 | Train Loss Unweighted: 1.4071 | Train Loss Components: Classification: 0.0009 | Classification_binary: 1.2068 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0938 | Regression_eta: 0.0060 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0114 | Classification_binary: 1.2068 | Regression_cos_phi: 0.0274 | Regression_energy: 0.5357 | Regression_eta: 0.0380 | Regression_pt: 0.7910 | Regression_sin_phi: 0.0313 | LR: 1.31e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5611s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0029 | max_backward: 0.0394 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1606 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.6 | uss_MB: 1585.3 | child_rss_MB: 22461.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:28:25,972] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 05:28:26,040] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:29:01,342] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:29:01,342] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:04,333] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:30:04,333] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:16,967] INFO : Step 49000/500000 rank0 | Train Loss: 0.9515 | Train Loss Unweighted: 0.5627 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.5009 | Regression_cos_phi: 0.0033 | Regression_energy: 0.0256 | Regression_eta: 0.0035 | Regression_pt: 0.0265 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0031 | Classification_binary: 0.5009 | Regression_cos_phi: 0.0306 | Regression_energy: 0.1462 | Regression_eta: 0.0221 | Regression_pt: 0.2254 | Regression_sin_phi: 0.0232 | LR: 1.36e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5881s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0387 | max_loss: 0.0031 | max_backward: 0.0408 | max_optimizer_step: 0.0700 | max_tensorboard_logging: 0.1742 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.6 | uss_MB: 1585.3 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 05:31:05,577] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:31:05,577] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:05,865] INFO : Step 50000/500000 rank0 | Train Loss: 1.7799 | Train Loss Unweighted: 0.7681 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.6068 | Regression_cos_phi: 0.0045 | Regression_energy: 0.0689 | Regression_eta: 0.0061 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0074 | Classification_binary: 0.6068 | Regression_cos_phi: 0.0420 | Regression_energy: 0.3933 | Regression_eta: 0.0386 | Regression_pt: 0.6446 | Regression_sin_phi: 0.0471 | LR: 1.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.6051s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0397 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.1900 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.4 | uss_MB: 1581.1 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:32:05,867] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:32:05,940] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:32:05,941] INFO : Running validation on rank0 +rank0 [2026-11-14 05:32:05,941] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:32:05,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:32:06,062] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:32:06,170] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:32:06,271] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:32:06,340] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:32:06,408] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:32:06,477] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:32:06,542] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:32:06,606] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:32:06,718] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:32:06,827] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:32:06,930] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:32:06,998] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:32:07,065] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:32:07,130] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:32:07,198] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:32:07,266] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:32:07,368] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:32:07,374] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:32:07,376] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:32:07,378] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:32:07,380] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:32:07,381] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:32:07,383] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:32:07,384] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:32:07,385] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:32:07,387] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:32:07,389] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:32:07,391] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:32:07,393] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:32:07,395] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:32:07,397] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:32:07,399] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:32:08,815] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.7799 | Valid Loss=10.0218 | Train Loss Unweighted=0.7681 | Valid Loss Unweighted=6.4135 | Stale=3 | ETA=834.5m +rank0 [2026-11-14 05:32:08,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:32:09,394] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:32:09,403] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:09,457] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:32:09,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:32:10,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:32:10,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:32:10,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:32:10,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:32:11,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:32:11,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:32:11,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:32:11,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:32:12,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:32:12,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:32:12,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:32:12,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:32:13,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:32:13,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:32:13,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:32:13,614] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:32:13,621] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:14,167] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:14,582] INFO : Plotted number of elements +rank0 [2026-11-14 05:32:15,866] INFO : Plotted jets +rank0 [2026-11-14 05:32:16,808] INFO : Plotted jet ratio +rank0 [2026-11-14 05:32:17,248] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:32:17,248] INFO : Training completed. Total time on device 0: 92.860min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..5d27d387a0b602db5d4de5a0ffcecd68fd5f262c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/train.log.0 @@ -0,0 +1,997 @@ +rank0 [2026-11-14 03:59:22,921] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 03:59:22,921] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 03:59:22,922] INFO : Instantiating model +rank0 [2026-11-14 03:59:22,953] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 03:59:22,953] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 03:59:22,955] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 03:59:22,955] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 03:59:22,955] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 03:59:22,955] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 03:59:22,955] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,958] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 03:59:22,958] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,961] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 03:59:22,961] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,964] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 03:59:22,964] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,966] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 03:59:22,966] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,969] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 03:59:22,969] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,972] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 03:59:22,972] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 03:59:22,972] INFO : Initializing output DNNs +rank0 [2026-11-14 03:59:22,979] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,982] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 03:59:22,990] INFO : Output DNNs initialization took 0.02s +rank0 [2026-11-14 03:59:22,990] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 03:59:22,990] INFO : jagged_attention enabled=True +rank0 [2026-11-14 03:59:22,990] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 03:59:22,990] INFO : task_queries enabled=True +rank0 [2026-11-14 03:59:22,990] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 03:59:22,990] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 03:59:22,990] INFO : classification_query parameters: 256 +rank0 [2026-11-14 03:59:22,990] INFO : regression_query parameters: 256 +rank0 [2026-11-14 03:59:22,990] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 03:59:22,990] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 03:59:22,991] INFO : output_mode=elementwise +rank0 [2026-11-14 03:59:22,991] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 03:59:22,991] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 03:59:22,991] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 03:59:22,991] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 03:59:22,991] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 03:59:22,991] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 03:59:22,991] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 03:59:22,991] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 03:59:22,992] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 03:59:22,992] INFO : MLPF __init__ done +rank0 [2026-11-14 03:59:22,993] INFO : Instantiated model +rank0 [2026-11-14 03:59:22,993] INFO : Moving model to device rank=0 +rank0 [2026-11-14 03:59:23,005] INFO : Moved model to device rank=0 +rank0 [2026-11-14 03:59:23,006] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 03:59:23,108] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 03:59:23,108] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 03:59:23,109] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 03:59:23,109] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 03:59:23,109] INFO : Total parameters: 3910930 +rank0 [2026-11-14 03:59:23,112] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 03:59:23,114] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072 +rank0 [2026-11-14 03:59:23,114] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072 +rank0 [2026-11-14 03:59:23,114] INFO : Getting dataloaders +rank0 [2026-11-14 03:59:23,114] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:59:24,978] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,009] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:59:25,052] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:59:25,115] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,144] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,189] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,253] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,294] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,335] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 03:59:25,366] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 03:59:25,366] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 03:59:25,367] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 03:59:25,367] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 03:59:25,367] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 03:59:25,375] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,385] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,394] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,403] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,410] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,418] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,425] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,432] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,440] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,447] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 03:59:25,447] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 03:59:25,447] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 03:59:25,500] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:59:25,647] INFO : Got dataloaders +rank0 [2026-11-14 03:59:25,647] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 03:59:25,647] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 03:59:25,648] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 03:59:25,653] INFO : Creating train_iterator +rank0 [2026-11-14 03:59:25,653] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 03:59:25,681] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 03:59:25,842] INFO : Created train_iterator +rank0 [2026-11-14 03:59:38,304] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.4912013244628906, 'Classification': 0.6057955461740494, 'Regression_pt': 0.8809010863304139, 'Regression_eta': 1.1775247964262963, 'Regression_sin_phi': 0.8549120762944221, 'Regression_cos_phi': 0.8112198185920715, 'Regression_energy': 1.311812883913517}, weights={'Classification_binary': 1.0, 'Classification': 12.365890502929688, 'Regression_pt': 8.504020690917969, 'Regression_eta': 6.361820220947266, 'Regression_sin_phi': 8.762539863586426, 'Regression_cos_phi': 9.234490394592285, 'Regression_energy': 5.710571765899658} +rank0 [2026-11-14 04:00:29,454] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 04:00:29,454] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:17,820] INFO : Step 1000/500000 rank0 | Train Loss: 9.9083 | Train Loss Unweighted: 4.5972 | Train Loss Components: Classification: 0.1353 | Classification_binary: 3.8487 | Regression_cos_phi: 0.0590 | Regression_energy: 0.1505 | Regression_eta: 0.2141 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0317 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 1.6736 | Classification_binary: 3.8487 | Regression_cos_phi: 0.5450 | Regression_energy: 0.8592 | Regression_eta: 1.3623 | Regression_pt: 1.3421 | Regression_sin_phi: 0.2774 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5831s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.2035 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2564.2 | uss_MB: 1405.1 | child_rss_MB: 22467.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:01:31,325] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:01:31,325] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:02:32,392] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:02:32,392] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:06,964] INFO : Step 2000/500000 rank0 | Train Loss: 5.7514 | Train Loss Unweighted: 3.4005 | Train Loss Components: Classification: 0.0472 | Classification_binary: 3.0614 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1272 | Regression_eta: 0.0144 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.5838 | Classification_binary: 3.0614 | Regression_cos_phi: 0.1054 | Regression_energy: 0.7266 | Regression_eta: 0.0915 | Regression_pt: 1.1182 | Regression_sin_phi: 0.0645 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5195s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0030 | max_backward: 0.0392 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1536 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2564.9 | uss_MB: 1405.9 | child_rss_MB: 22460.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:03:06,969] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:03:34,000] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:03:34,001] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:35,529] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:04:35,529] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:56,742] INFO : Step 3000/500000 rank0 | Train Loss: 8.9234 | Train Loss Unweighted: 3.5631 | Train Loss Components: Classification: 0.0387 | Classification_binary: 2.7249 | Regression_cos_phi: 0.0256 | Regression_energy: 0.3390 | Regression_eta: 0.0718 | Regression_pt: 0.3497 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.4789 | Classification_binary: 2.7249 | Regression_cos_phi: 0.2363 | Regression_energy: 1.9359 | Regression_eta: 0.4569 | Regression_pt: 2.9740 | Regression_sin_phi: 0.1166 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5217s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0392 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1338 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2595.5 | uss_MB: 1437.0 | child_rss_MB: 22459.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:05:37,567] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:05:37,567] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:38,909] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:06:38,909] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:46,555] INFO : Step 4000/500000 rank0 | Train Loss: 8.3284 | Train Loss Unweighted: 4.2501 | Train Loss Components: Classification: 0.0321 | Classification_binary: 3.6189 | Regression_cos_phi: 0.0094 | Regression_energy: 0.2708 | Regression_eta: 0.0186 | Regression_pt: 0.2700 | Regression_sin_phi: 0.0301 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.3971 | Classification_binary: 3.6189 | Regression_cos_phi: 0.0869 | Regression_energy: 1.5463 | Regression_eta: 0.1186 | Regression_pt: 2.2965 | Regression_sin_phi: 0.2640 | LR: 2.08e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5251s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0395 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2595.6 | uss_MB: 1437.0 | child_rss_MB: 22459.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:06:46,559] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:07:41,032] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:07:41,032] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:37,065] INFO : Step 5000/500000 rank0 | Train Loss: 5.4880 | Train Loss Unweighted: 3.2653 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.9225 | Regression_cos_phi: 0.0153 | Regression_energy: 0.1422 | Regression_eta: 0.0170 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2227 | Classification_binary: 2.9225 | Regression_cos_phi: 0.1416 | Regression_energy: 0.8121 | Regression_eta: 0.1082 | Regression_pt: 1.1581 | Regression_sin_phi: 0.1228 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5201s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2596.8 | uss_MB: 1438.3 | child_rss_MB: 22460.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:08:43,404] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:08:43,404] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:44,780] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:09:44,780] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:26,850] INFO : Step 6000/500000 rank0 | Train Loss: 11.7303 | Train Loss Unweighted: 4.6051 | Train Loss Components: Classification: 0.0208 | Classification_binary: 3.4784 | Regression_cos_phi: 0.1495 | Regression_energy: 0.4900 | Regression_eta: 0.0768 | Regression_pt: 0.3359 | Regression_sin_phi: 0.0537 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2575 | Classification_binary: 3.4784 | Regression_cos_phi: 1.3801 | Regression_energy: 2.7984 | Regression_eta: 0.4883 | Regression_pt: 2.8567 | Regression_sin_phi: 0.4709 | LR: 2.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5455s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0031 | max_backward: 0.0405 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.1388 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2596.8 | uss_MB: 1438.4 | child_rss_MB: 22459.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:10:26,852] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:10:46,981] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:10:46,982] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:48,904] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:11:48,904] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:12:17,546] INFO : Step 7000/500000 rank0 | Train Loss: 9.0172 | Train Loss Unweighted: 3.9994 | Train Loss Components: Classification: 0.0132 | Classification_binary: 3.2114 | Regression_cos_phi: 0.0085 | Regression_energy: 0.3278 | Regression_eta: 0.0179 | Regression_pt: 0.4163 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1636 | Classification_binary: 3.2114 | Regression_cos_phi: 0.0788 | Regression_energy: 1.8718 | Regression_eta: 0.1138 | Regression_pt: 3.5398 | Regression_sin_phi: 0.0379 | LR: 2.26e-05 | DataLoad Time: 0.0049s | Model Forward Time: 0.4844s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1211 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2596.9 | uss_MB: 1438.5 | child_rss_MB: 22459.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:12:51,702] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:12:51,702] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:54,115] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:13:54,116] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:09,210] INFO : Step 8000/500000 rank0 | Train Loss: 5.8763 | Train Loss Unweighted: 2.6389 | Train Loss Components: Classification: 0.0242 | Classification_binary: 2.1372 | Regression_cos_phi: 0.0200 | Regression_energy: 0.1916 | Regression_eta: 0.0477 | Regression_pt: 0.2105 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2997 | Classification_binary: 2.1372 | Regression_cos_phi: 0.1848 | Regression_energy: 1.0944 | Regression_eta: 0.3036 | Regression_pt: 1.7898 | Regression_sin_phi: 0.0668 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5587s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0380 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.1633 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2597.0 | uss_MB: 1438.5 | child_rss_MB: 22458.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:14:09,212] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:14:09,282] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:14:56,698] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:14:56,698] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:57,878] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:15:57,878] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:59,127] INFO : Step 9000/500000 rank0 | Train Loss: 7.9815 | Train Loss Unweighted: 2.9420 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.1436 | Regression_cos_phi: 0.0093 | Regression_energy: 0.3622 | Regression_eta: 0.0160 | Regression_pt: 0.3847 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2783 | Classification_binary: 2.1436 | Regression_cos_phi: 0.0855 | Regression_energy: 2.0683 | Regression_eta: 0.1017 | Regression_pt: 3.2714 | Regression_sin_phi: 0.0326 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5028s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0336 | max_loss: 0.0024 | max_backward: 0.0374 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1527 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2565.1 | uss_MB: 1406.6 | child_rss_MB: 22458.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:17:00,362] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:17:00,363] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:17:49,511] INFO : Step 10000/500000 rank0 | Train Loss: 4.5065 | Train Loss Unweighted: 1.8686 | Train Loss Components: Classification: 0.0123 | Classification_binary: 1.4506 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1930 | Regression_eta: 0.0055 | Regression_pt: 0.1967 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1526 | Classification_binary: 1.4506 | Regression_cos_phi: 0.0450 | Regression_energy: 1.1023 | Regression_eta: 0.0348 | Regression_pt: 1.6727 | Regression_sin_phi: 0.0485 | LR: 2.52e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5296s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1765 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2565.6 | uss_MB: 1407.1 | child_rss_MB: 22459.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:17:49,513] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:17:49,583] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:17:49,584] INFO : Running validation on rank0 +rank0 [2026-11-14 04:17:49,584] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:17:49,584] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:17:49,789] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:17:50,025] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:17:50,095] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:17:50,151] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:17:50,206] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:17:50,253] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:17:50,308] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:17:50,363] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:17:50,411] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:17:50,486] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:17:50,556] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:17:50,611] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:17:50,664] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:17:50,710] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:17:50,762] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:17:50,815] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:17:50,865] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:17:50,869] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:17:50,871] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:17:50,873] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:17:50,875] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:17:50,877] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:17:50,877] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:17:50,879] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:17:50,881] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:17:50,882] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:17:50,883] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:17:50,884] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:17:50,887] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:17:50,889] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:17:50,891] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:17:50,893] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:17:52,334] INFO : VALIDATION | Step=10000/500000 | Train Loss=4.5065 | Valid Loss=6.3588 | Train Loss Unweighted=1.8686 | Valid Loss Unweighted=2.9581 | Stale=0 | ETA=903.8m +rank0 [2026-11-14 04:17:52,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:17:52,939] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:17:52,946] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:53,000] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:17:53,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:17:53,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:17:54,221] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:17:54,456] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:17:54,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:17:54,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:17:55,171] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:17:55,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:17:55,656] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:17:55,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:17:56,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:17:56,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:17:56,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:17:56,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:17:57,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:17:57,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:17:57,358] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:17:57,366] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:57,898] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:17:58,796] INFO : Plotted number of elements +rank0 [2026-11-14 04:18:00,060] INFO : Plotted jets +rank0 [2026-11-14 04:18:00,990] INFO : Plotted jet ratio +rank0 [2026-11-14 04:18:13,669] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:18:13,669] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:15,209] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:19:15,209] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:51,337] INFO : Step 11000/500000 rank0 | Train Loss: 3.2896 | Train Loss Unweighted: 1.7410 | Train Loss Components: Classification: 0.0140 | Classification_binary: 1.5015 | Regression_cos_phi: 0.0036 | Regression_energy: 0.1052 | Regression_eta: 0.0061 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1726 | Classification_binary: 1.5015 | Regression_cos_phi: 0.0331 | Regression_energy: 0.6010 | Regression_eta: 0.0389 | Regression_pt: 0.8697 | Regression_sin_phi: 0.0728 | LR: 2.63e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5321s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1339 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2702.2 | uss_MB: 1556.9 | child_rss_MB: 22461.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:20:17,835] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:20:17,835] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:19,692] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:21:19,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:41,699] INFO : Step 12000/500000 rank0 | Train Loss: 3.8566 | Train Loss Unweighted: 2.5132 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.3122 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0827 | Regression_eta: 0.0065 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2393 | Classification_binary: 2.3122 | Regression_cos_phi: 0.0456 | Regression_energy: 0.4720 | Regression_eta: 0.0416 | Regression_pt: 0.6974 | Regression_sin_phi: 0.0485 | LR: 2.75e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5130s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0342 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0640 | max_tensorboard_logging: 0.1472 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2702.2 | uss_MB: 1556.9 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:21:41,701] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:21:41,770] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:22:21,423] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:22:21,423] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:22,964] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:23:22,964] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:31,436] INFO : Step 13000/500000 rank0 | Train Loss: 8.5581 | Train Loss Unweighted: 3.3389 | Train Loss Components: Classification: 0.0064 | Classification_binary: 2.4967 | Regression_cos_phi: 0.0058 | Regression_energy: 0.4022 | Regression_eta: 0.0049 | Regression_pt: 0.4060 | Regression_sin_phi: 0.0169 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0788 | Classification_binary: 2.4967 | Regression_cos_phi: 0.0536 | Regression_energy: 2.2970 | Regression_eta: 0.0312 | Regression_pt: 3.4524 | Regression_sin_phi: 0.1484 | LR: 2.88e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5075s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0641 | max_tensorboard_logging: 0.1490 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2645.1 | uss_MB: 1499.8 | child_rss_MB: 22461.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:24,341] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:24:24,341] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:20,419] INFO : Step 14000/500000 rank0 | Train Loss: 5.4174 | Train Loss Unweighted: 2.1849 | Train Loss Components: Classification: 0.0064 | Classification_binary: 1.6464 | Regression_cos_phi: 0.0059 | Regression_energy: 0.2917 | Regression_eta: 0.0115 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0788 | Classification_binary: 1.6464 | Regression_cos_phi: 0.0546 | Regression_energy: 1.6657 | Regression_eta: 0.0731 | Regression_pt: 1.8145 | Regression_sin_phi: 0.0844 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5188s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0032 | max_backward: 0.0404 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1358 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2645.7 | uss_MB: 1500.4 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:25:20,421] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:25:20,489] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:25:25,905] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:25:25,905] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:26,933] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:26:26,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:09,928] INFO : Step 15000/500000 rank0 | Train Loss: 3.5910 | Train Loss Unweighted: 2.4022 | Train Loss Components: Classification: 0.0117 | Classification_binary: 2.2218 | Regression_cos_phi: 0.0066 | Regression_energy: 0.0709 | Regression_eta: 0.0085 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1444 | Classification_binary: 2.2218 | Regression_cos_phi: 0.0610 | Regression_energy: 0.4050 | Regression_eta: 0.0540 | Regression_pt: 0.6630 | Regression_sin_phi: 0.0418 | LR: 3.17e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5367s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1669 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2647.4 | uss_MB: 1502.1 | child_rss_MB: 22461.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:27:28,727] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:27:28,727] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:29,721] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:28:29,722] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:58,838] INFO : Step 16000/500000 rank0 | Train Loss: 2.1657 | Train Loss Unweighted: 1.2431 | Train Loss Components: Classification: 0.0016 | Classification_binary: 1.0977 | Regression_cos_phi: 0.0086 | Regression_energy: 0.0589 | Regression_eta: 0.0080 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0198 | Classification_binary: 1.0977 | Regression_cos_phi: 0.0797 | Regression_energy: 0.3364 | Regression_eta: 0.0507 | Regression_pt: 0.5606 | Regression_sin_phi: 0.0207 | LR: 3.33e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5485s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0030 | max_backward: 0.0397 | max_optimizer_step: 0.0646 | max_tensorboard_logging: 0.1539 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2647.5 | uss_MB: 1502.2 | child_rss_MB: 22459.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:28:58,840] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:58,909] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:29:31,237] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:29:31,237] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:32,003] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:30:32,003] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:47,739] INFO : Step 17000/500000 rank0 | Train Loss: 4.3476 | Train Loss Unweighted: 2.5891 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3193 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1155 | Regression_eta: 0.0096 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2244 | Classification_binary: 2.3193 | Regression_cos_phi: 0.0621 | Regression_energy: 0.6593 | Regression_eta: 0.0613 | Regression_pt: 0.9461 | Regression_sin_phi: 0.0750 | LR: 3.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5310s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.1339 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2645.1 | uss_MB: 1499.8 | child_rss_MB: 22459.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:31:34,168] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:31:34,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:35,462] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:32:35,462] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:37,657] INFO : Step 18000/500000 rank0 | Train Loss: 2.5472 | Train Loss Unweighted: 1.6154 | Train Loss Components: Classification: 0.0299 | Classification_binary: 1.4904 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0421 | Regression_eta: 0.0038 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.3699 | Classification_binary: 1.4904 | Regression_cos_phi: 0.0316 | Regression_energy: 0.2402 | Regression_eta: 0.0239 | Regression_pt: 0.3665 | Regression_sin_phi: 0.0246 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5379s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0027 | max_backward: 0.0381 | max_optimizer_step: 0.0646 | max_tensorboard_logging: 0.1627 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2646.2 | uss_MB: 1500.9 | child_rss_MB: 22461.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:32:37,659] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:32:37,729] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:33:37,167] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:33:37,167] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:27,254] INFO : Step 19000/500000 rank0 | Train Loss: 5.9953 | Train Loss Unweighted: 3.6054 | Train Loss Components: Classification: 0.0224 | Classification_binary: 3.2414 | Regression_cos_phi: 0.0172 | Regression_energy: 0.1512 | Regression_eta: 0.0091 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2767 | Classification_binary: 3.2414 | Regression_cos_phi: 0.1590 | Regression_energy: 0.8634 | Regression_eta: 0.0581 | Regression_pt: 1.3402 | Regression_sin_phi: 0.0565 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5494s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1568 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2647.7 | uss_MB: 1502.4 | child_rss_MB: 22460.7 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:34:39,051] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:34:39,052] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:40,616] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:35:40,616] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:17,804] INFO : Step 20000/500000 rank0 | Train Loss: 2.7474 | Train Loss Unweighted: 1.4205 | Train Loss Components: Classification: 0.0056 | Classification_binary: 1.2125 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0898 | Regression_eta: 0.0046 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 1.2125 | Regression_cos_phi: 0.0337 | Regression_energy: 0.5129 | Regression_eta: 0.0290 | Regression_pt: 0.8134 | Regression_sin_phi: 0.0766 | LR: 4.07e-05 | DataLoad Time: 0.0063s | Model Forward Time: 0.5333s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0031 | max_backward: 0.0407 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1562 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2648.0 | uss_MB: 1502.7 | child_rss_MB: 22460.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:36:17,806] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:36:17,872] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:36:17,873] INFO : Running validation on rank0 +rank0 [2026-11-14 04:36:17,873] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:36:17,873] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:36:17,989] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:36:18,101] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:36:18,206] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:36:18,275] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:36:18,341] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:36:18,405] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:36:18,474] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:36:18,544] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:36:18,646] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:36:18,752] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:36:18,856] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:36:18,923] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:36:18,989] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:36:19,054] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:36:19,121] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:36:19,189] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:36:19,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:36:19,312] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:36:19,315] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:36:19,317] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:36:19,319] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:36:19,321] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:36:19,322] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:36:19,323] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:36:19,325] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:36:19,326] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:36:19,328] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:36:19,330] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:36:19,332] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:36:19,334] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:36:19,335] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:36:19,338] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:36:20,831] INFO : VALIDATION | Step=20000/500000 | Train Loss=2.7474 | Valid Loss=5.7297 | Train Loss Unweighted=1.4205 | Valid Loss Unweighted=2.7609 | Stale=0 | ETA=886.1m +rank0 [2026-11-14 04:36:20,831] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:36:21,380] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:36:21,387] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:36:21,438] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:36:21,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:36:22,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:36:22,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:36:22,655] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:36:22,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:36:23,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:36:23,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:36:23,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:36:23,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:36:24,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:36:24,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:36:24,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:36:24,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:36:25,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:36:25,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:36:25,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:36:25,573] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:36:25,580] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:36:26,127] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:36:26,533] INFO : Plotted number of elements +rank0 [2026-11-14 04:36:27,791] INFO : Plotted jets +rank0 [2026-11-14 04:36:28,734] INFO : Plotted jet ratio +rank0 [2026-11-14 04:36:54,747] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:36:54,747] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:56,528] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:37:56,528] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:38:19,589] INFO : Step 21000/500000 rank0 | Train Loss: 3.9015 | Train Loss Unweighted: 2.5558 | Train Loss Components: Classification: 0.0064 | Classification_binary: 2.3400 | Regression_cos_phi: 0.0035 | Regression_energy: 0.1040 | Regression_eta: 0.0053 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 2.3400 | Regression_cos_phi: 0.0324 | Regression_energy: 0.5941 | Regression_eta: 0.0340 | Regression_pt: 0.7973 | Regression_sin_phi: 0.0246 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5364s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0027 | max_backward: 0.0443 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1435 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2675.5 | uss_MB: 1530.2 | child_rss_MB: 22461.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:38:58,646] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:38:58,646] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:00,474] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:40:00,474] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:09,514] INFO : Step 22000/500000 rank0 | Train Loss: 3.0897 | Train Loss Unweighted: 1.8899 | Train Loss Components: Classification: 0.0102 | Classification_binary: 1.7062 | Regression_cos_phi: 0.0015 | Regression_energy: 0.0721 | Regression_eta: 0.0088 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1258 | Classification_binary: 1.7062 | Regression_cos_phi: 0.0142 | Regression_energy: 0.4119 | Regression_eta: 0.0558 | Regression_pt: 0.7336 | Regression_sin_phi: 0.0422 | LR: 4.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4924s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.1501 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2676.3 | uss_MB: 1531.0 | child_rss_MB: 22461.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:40:09,515] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:40:09,583] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:41:02,892] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:41:02,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:00,918] INFO : Step 23000/500000 rank0 | Train Loss: 3.7919 | Train Loss Unweighted: 2.5744 | Train Loss Components: Classification: 0.0105 | Classification_binary: 2.3864 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0807 | Regression_eta: 0.0056 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1303 | Classification_binary: 2.3864 | Regression_cos_phi: 0.0400 | Regression_energy: 0.4611 | Regression_eta: 0.0354 | Regression_pt: 0.7060 | Regression_sin_phi: 0.0327 | LR: 4.73e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0029 | max_backward: 0.0391 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1490 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2678.6 | uss_MB: 1533.3 | child_rss_MB: 22460.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:42:05,371] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:42:05,371] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:05,993] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:43:05,993] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:50,110] INFO : Step 24000/500000 rank0 | Train Loss: 7.1134 | Train Loss Unweighted: 2.5597 | Train Loss Components: Classification: 0.0063 | Classification_binary: 1.8237 | Regression_cos_phi: 0.0067 | Regression_energy: 0.3491 | Regression_eta: 0.0123 | Regression_pt: 0.3505 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 1.8237 | Regression_cos_phi: 0.0620 | Regression_energy: 1.9937 | Regression_eta: 0.0782 | Regression_pt: 2.9808 | Regression_sin_phi: 0.0969 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5703s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0700 | max_tensorboard_logging: 0.1830 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.6 | uss_MB: 1533.3 | child_rss_MB: 22461.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:43:50,112] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:43:50,183] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:44:08,397] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:44:08,397] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:10,609] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:45:10,609] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:41,759] INFO : Step 25000/500000 rank0 | Train Loss: 4.1535 | Train Loss Unweighted: 2.3185 | Train Loss Components: Classification: 0.0081 | Classification_binary: 2.0293 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1287 | Regression_eta: 0.0074 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1005 | Classification_binary: 2.0293 | Regression_cos_phi: 0.0876 | Regression_energy: 0.7348 | Regression_eta: 0.0470 | Regression_pt: 1.0891 | Regression_sin_phi: 0.0651 | LR: 5.22e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5149s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0367 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0700 | max_tensorboard_logging: 0.1206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2675.6 | uss_MB: 1530.3 | child_rss_MB: 22460.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:46:14,015] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:46:14,015] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:16,337] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:47:16,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:33,133] INFO : Step 26000/500000 rank0 | Train Loss: 2.6816 | Train Loss Unweighted: 1.7997 | Train Loss Components: Classification: 0.0025 | Classification_binary: 1.6599 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0595 | Regression_eta: 0.0066 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 1.6599 | Regression_cos_phi: 0.0360 | Regression_energy: 0.3397 | Regression_eta: 0.0420 | Regression_pt: 0.5390 | Regression_sin_phi: 0.0337 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5572s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0031 | max_backward: 0.0395 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1530 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2676.4 | uss_MB: 1531.1 | child_rss_MB: 22462.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:47:33,135] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:47:33,208] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:48:18,897] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:48:18,897] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:20,965] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:49:20,965] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:23,998] INFO : Step 27000/500000 rank0 | Train Loss: 2.7217 | Train Loss Unweighted: 1.6031 | Train Loss Components: Classification: 0.0028 | Classification_binary: 1.4216 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0839 | Regression_eta: 0.0113 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 1.4216 | Regression_cos_phi: 0.0311 | Regression_energy: 0.4792 | Regression_eta: 0.0717 | Regression_pt: 0.6254 | Regression_sin_phi: 0.0582 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5158s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1212 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.3 | uss_MB: 1533.0 | child_rss_MB: 22462.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:50:23,447] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:50:23,447] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:14,554] INFO : Step 28000/500000 rank0 | Train Loss: 2.7285 | Train Loss Unweighted: 1.6226 | Train Loss Components: Classification: 0.0137 | Classification_binary: 1.4558 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0610 | Regression_eta: 0.0153 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1690 | Classification_binary: 1.4558 | Regression_cos_phi: 0.0447 | Regression_energy: 0.3482 | Regression_eta: 0.0975 | Regression_pt: 0.5502 | Regression_sin_phi: 0.0631 | LR: 6.01e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5268s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0783 | max_tensorboard_logging: 0.1239 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.5 | uss_MB: 1533.2 | child_rss_MB: 22461.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:51:14,556] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:51:14,623] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:51:25,568] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:51:25,568] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:27,151] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:52:27,152] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:04,891] INFO : Step 29000/500000 rank0 | Train Loss: 2.8718 | Train Loss Unweighted: 1.5655 | Train Loss Components: Classification: 0.0093 | Classification_binary: 1.3615 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0912 | Regression_eta: 0.0045 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1148 | Classification_binary: 1.3615 | Regression_cos_phi: 0.0242 | Regression_energy: 0.5210 | Regression_eta: 0.0287 | Regression_pt: 0.7577 | Regression_sin_phi: 0.0640 | LR: 6.29e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5237s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0744 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.5 | uss_MB: 1533.2 | child_rss_MB: 22461.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:53:29,550] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:53:29,553] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:31,269] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:54:31,269] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:55,077] INFO : Step 30000/500000 rank0 | Train Loss: 3.1105 | Train Loss Unweighted: 1.6862 | Train Loss Components: Classification: 0.0015 | Classification_binary: 1.4559 | Regression_cos_phi: 0.0021 | Regression_energy: 0.1086 | Regression_eta: 0.0041 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0191 | Classification_binary: 1.4559 | Regression_cos_phi: 0.0197 | Regression_energy: 0.6203 | Regression_eta: 0.0263 | Regression_pt: 0.9421 | Regression_sin_phi: 0.0271 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5188s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0383 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2676.4 | uss_MB: 1531.1 | child_rss_MB: 22462.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:54:55,079] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:54:55,146] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:54:55,147] INFO : Running validation on rank0 +rank0 [2026-11-14 04:54:55,147] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:54:55,147] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:54:55,274] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:54:55,384] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:54:55,488] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:54:55,556] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:54:55,624] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:54:55,689] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:54:55,755] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:54:55,821] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:54:55,923] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:54:56,032] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:54:56,142] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:54:56,209] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:54:56,276] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:54:56,342] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:54:56,408] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:54:56,475] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:54:56,577] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:54:56,583] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:54:56,585] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:54:56,588] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:54:56,590] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:54:56,591] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:54:56,592] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:54:56,594] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:54:56,596] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:54:56,597] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:54:56,598] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:54:56,600] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:54:56,602] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:54:56,604] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:54:56,606] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:54:56,608] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:54:58,040] INFO : VALIDATION | Step=30000/500000 | Train Loss=3.1105 | Valid Loss=5.8660 | Train Loss Unweighted=1.6862 | Valid Loss Unweighted=3.0290 | Stale=1 | ETA=870.1m +rank0 [2026-11-14 04:54:58,040] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:54:58,554] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:54:58,563] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:58,611] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:54:59,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:54:59,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:54:59,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:54:59,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:55:00,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:55:00,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:55:00,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:55:00,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:55:01,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:55:01,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:55:01,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:55:01,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:55:01,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:55:02,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:55:02,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:55:02,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:55:02,759] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:55:02,765] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:55:03,304] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:55:03,716] INFO : Plotted number of elements +rank0 [2026-11-14 04:55:04,989] INFO : Plotted jets +rank0 [2026-11-14 04:55:05,938] INFO : Plotted jet ratio +rank0 [2026-11-14 04:55:44,662] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:55:44,662] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:46,562] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:56:46,563] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:56,907] INFO : Step 31000/500000 rank0 | Train Loss: 2.8743 | Train Loss Unweighted: 1.9270 | Train Loss Components: Classification: 0.0106 | Classification_binary: 1.7826 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0609 | Regression_eta: 0.0048 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.1307 | Classification_binary: 1.7826 | Regression_cos_phi: 0.0317 | Regression_energy: 0.3479 | Regression_eta: 0.0308 | Regression_pt: 0.5172 | Regression_sin_phi: 0.0335 | LR: 6.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5363s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0032 | max_backward: 0.0403 | max_optimizer_step: 0.0714 | max_tensorboard_logging: 0.1221 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2793.2 | uss_MB: 1648.0 | child_rss_MB: 22461.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:57:49,032] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:57:49,032] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:47,702] INFO : Step 32000/500000 rank0 | Train Loss: 3.9157 | Train Loss Unweighted: 2.6112 | Train Loss Components: Classification: 0.0032 | Classification_binary: 2.3997 | Regression_cos_phi: 0.0041 | Regression_energy: 0.1018 | Regression_eta: 0.0070 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 2.3997 | Regression_cos_phi: 0.0377 | Regression_energy: 0.5811 | Regression_eta: 0.0446 | Regression_pt: 0.7730 | Regression_sin_phi: 0.0399 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5120s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0396 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1236 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2793.2 | uss_MB: 1648.0 | child_rss_MB: 22461.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:58:47,704] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:58:47,773] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:58:51,515] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:58:51,515] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:53,626] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:59:53,627] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:38,454] INFO : Step 33000/500000 rank0 | Train Loss: 4.4119 | Train Loss Unweighted: 1.7205 | Train Loss Components: Classification: 0.0067 | Classification_binary: 1.2896 | Regression_cos_phi: 0.0075 | Regression_energy: 0.2007 | Regression_eta: 0.0066 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 1.2896 | Regression_cos_phi: 0.0694 | Regression_energy: 1.1459 | Regression_eta: 0.0422 | Regression_pt: 1.7336 | Regression_sin_phi: 0.0480 | LR: 7.51e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5115s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0380 | max_loss: 0.0033 | max_backward: 0.0388 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1315 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2701.6 | uss_MB: 1556.3 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:00:55,744] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:00:55,744] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:57,701] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:01:57,701] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:28,799] INFO : Step 34000/500000 rank0 | Train Loss: 3.3924 | Train Loss Unweighted: 1.5938 | Train Loss Components: Classification: 0.0042 | Classification_binary: 1.3043 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1379 | Regression_eta: 0.0033 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 1.3043 | Regression_cos_phi: 0.0143 | Regression_energy: 0.7873 | Regression_eta: 0.0207 | Regression_pt: 1.1908 | Regression_sin_phi: 0.0228 | LR: 7.83e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5289s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0405 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1318 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2702.1 | uss_MB: 1556.8 | child_rss_MB: 22463.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:02:28,801] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:02:28,871] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:02:59,789] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:02:59,789] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:01,481] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:04:01,481] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:18,771] INFO : Step 35000/500000 rank0 | Train Loss: 1.1370 | Train Loss Unweighted: 0.5912 | Train Loss Components: Classification: 0.0028 | Classification_binary: 0.5060 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0337 | Regression_eta: 0.0059 | Regression_pt: 0.0373 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.5060 | Regression_cos_phi: 0.0250 | Regression_energy: 0.1922 | Regression_eta: 0.0375 | Regression_pt: 0.3171 | Regression_sin_phi: 0.0247 | LR: 8.16e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5029s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0027 | max_backward: 0.0390 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1516 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2704.6 | uss_MB: 1559.3 | child_rss_MB: 22462.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:05:02,463] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:05:02,463] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:05,555] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:06:05,556] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:09,388] INFO : Step 36000/500000 rank0 | Train Loss: 3.3995 | Train Loss Unweighted: 1.9744 | Train Loss Components: Classification: 0.0061 | Classification_binary: 1.7477 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1038 | Regression_eta: 0.0066 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 1.7477 | Regression_cos_phi: 0.0418 | Regression_energy: 0.5927 | Regression_eta: 0.0418 | Regression_pt: 0.8678 | Regression_sin_phi: 0.0324 | LR: 8.50e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5484s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0382 | max_optimizer_step: 0.0645 | max_tensorboard_logging: 0.1623 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.6 | uss_MB: 1559.4 | child_rss_MB: 22461.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:06:09,390] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:06:09,459] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:07:05,606] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:07:05,606] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:57,921] INFO : Step 37000/500000 rank0 | Train Loss: 2.4504 | Train Loss Unweighted: 1.5685 | Train Loss Components: Classification: 0.0014 | Classification_binary: 1.4253 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0647 | Regression_eta: 0.0100 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0168 | Classification_binary: 1.4253 | Regression_cos_phi: 0.0472 | Regression_energy: 0.3696 | Regression_eta: 0.0639 | Regression_pt: 0.5050 | Regression_sin_phi: 0.0225 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5338s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0030 | max_backward: 0.0410 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2704.7 | uss_MB: 1559.4 | child_rss_MB: 22461.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:08:08,023] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:08:08,023] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:10,229] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:09:10,229] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:48,688] INFO : Step 38000/500000 rank0 | Train Loss: 2.0249 | Train Loss Unweighted: 1.0969 | Train Loss Components: Classification: 0.0077 | Classification_binary: 0.9528 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0630 | Regression_eta: 0.0050 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0955 | Classification_binary: 0.9528 | Regression_cos_phi: 0.0255 | Regression_energy: 0.3597 | Regression_eta: 0.0318 | Regression_pt: 0.5129 | Regression_sin_phi: 0.0467 | LR: 9.21e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6119s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0028 | max_backward: 0.0387 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.2231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2702.4 | uss_MB: 1557.2 | child_rss_MB: 22462.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:09:48,690] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:09:48,761] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:10:12,578] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:10:12,579] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:14,280] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:11:14,280] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:39,105] INFO : Step 39000/500000 rank0 | Train Loss: 2.0043 | Train Loss Unweighted: 1.1855 | Train Loss Components: Classification: 0.0013 | Classification_binary: 1.0551 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0547 | Regression_eta: 0.0082 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0156 | Classification_binary: 1.0551 | Regression_cos_phi: 0.0493 | Regression_energy: 0.3125 | Regression_eta: 0.0520 | Regression_pt: 0.4751 | Regression_sin_phi: 0.0446 | LR: 9.57e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5248s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0030 | max_backward: 0.0390 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1366 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2704.9 | uss_MB: 1559.7 | child_rss_MB: 22461.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:12:16,164] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:12:16,165] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:18,719] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:13:18,719] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:30,102] INFO : Step 40000/500000 rank0 | Train Loss: 1.6980 | Train Loss Unweighted: 0.8736 | Train Loss Components: Classification: 0.0197 | Classification_binary: 0.7564 | Regression_cos_phi: 0.0021 | Regression_energy: 0.0441 | Regression_eta: 0.0046 | Regression_pt: 0.0439 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.2432 | Classification_binary: 0.7564 | Regression_cos_phi: 0.0191 | Regression_energy: 0.2519 | Regression_eta: 0.0294 | Regression_pt: 0.3736 | Regression_sin_phi: 0.0242 | LR: 9.94e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5569s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0030 | max_backward: 0.0396 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1550 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2705.1 | uss_MB: 1559.8 | child_rss_MB: 22460.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:13:30,104] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:13:30,170] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:13:30,171] INFO : Running validation on rank0 +rank0 [2026-11-14 05:13:30,171] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:13:30,171] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:13:30,290] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:13:30,396] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:13:30,494] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:13:30,563] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:13:30,630] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:13:30,695] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:13:30,761] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:13:30,829] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:13:30,942] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:13:31,058] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:13:31,172] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:13:31,242] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:13:31,311] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:13:31,379] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:13:31,443] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:13:31,511] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:13:31,609] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:13:31,615] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:13:31,617] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:13:31,619] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:13:31,620] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:13:31,623] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:13:31,624] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:13:31,626] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:13:31,628] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:13:31,629] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:13:31,630] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:13:31,632] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:13:31,634] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:13:31,636] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:13:31,638] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:13:31,640] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:13:33,074] INFO : VALIDATION | Step=40000/500000 | Train Loss=1.6980 | Valid Loss=8.4073 | Train Loss Unweighted=0.8736 | Valid Loss Unweighted=4.9177 | Stale=2 | ETA=852.4m +rank0 [2026-11-14 05:13:33,075] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:13:33,546] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:13:33,554] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:33,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:13:34,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:13:34,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:13:34,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:13:34,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:13:35,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:13:35,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:13:35,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:13:35,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:13:36,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:13:36,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:13:36,581] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:13:36,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:13:37,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:13:37,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:13:37,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:13:37,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:13:37,826] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:13:37,832] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:38,384] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:13:38,788] INFO : Plotted number of elements +rank0 [2026-11-14 05:13:40,079] INFO : Plotted jets +rank0 [2026-11-14 05:13:41,043] INFO : Plotted jet ratio +rank0 [2026-11-14 05:14:32,812] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:14:32,812] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:32,277] INFO : Step 41000/500000 rank0 | Train Loss: 1.2421 | Train Loss Unweighted: 0.5587 | Train Loss Components: Classification: 0.0010 | Classification_binary: 0.4492 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0468 | Regression_eta: 0.0066 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0128 | Classification_binary: 0.4492 | Regression_cos_phi: 0.0285 | Regression_energy: 0.2674 | Regression_eta: 0.0422 | Regression_pt: 0.4020 | Regression_sin_phi: 0.0400 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5483s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0031 | max_backward: 0.0397 | max_optimizer_step: 0.0712 | max_tensorboard_logging: 0.1393 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.2 | uss_MB: 1581.0 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:15:35,097] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:15:35,097] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:36,613] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:16:36,613] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:22,268] INFO : Step 42000/500000 rank0 | Train Loss: 1.9269 | Train Loss Unweighted: 1.0489 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.9050 | Regression_cos_phi: 0.0024 | Regression_energy: 0.0645 | Regression_eta: 0.0118 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0052 | Classification_binary: 0.9050 | Regression_cos_phi: 0.0226 | Regression_energy: 0.3681 | Regression_eta: 0.0754 | Regression_pt: 0.5343 | Regression_sin_phi: 0.0164 | LR: 1.07e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5923s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0392 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1958 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.7 | uss_MB: 1581.5 | child_rss_MB: 22462.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:17:22,270] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:17:22,341] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:17:38,753] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:17:38,753] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:41,099] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:18:41,099] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:13,310] INFO : Step 43000/500000 rank0 | Train Loss: 1.7223 | Train Loss Unweighted: 0.7572 | Train Loss Components: Classification: 0.0042 | Classification_binary: 0.6069 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0579 | Regression_eta: 0.0114 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0172 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.6069 | Regression_cos_phi: 0.0396 | Regression_energy: 0.3304 | Regression_eta: 0.0725 | Regression_pt: 0.4699 | Regression_sin_phi: 0.1509 | LR: 1.11e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5376s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1363 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.5 | uss_MB: 1585.3 | child_rss_MB: 22462.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:19:43,456] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:19:43,456] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:45,108] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:20:45,108] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:03,550] INFO : Step 44000/500000 rank0 | Train Loss: 0.8604 | Train Loss Unweighted: 0.4138 | Train Loss Components: Classification: 0.0010 | Classification_binary: 0.3417 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0325 | Regression_eta: 0.0044 | Regression_pt: 0.0299 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0121 | Classification_binary: 0.3417 | Regression_cos_phi: 0.0189 | Regression_energy: 0.1854 | Regression_eta: 0.0279 | Regression_pt: 0.2539 | Regression_sin_phi: 0.0205 | LR: 1.15e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5495s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0638 | max_tensorboard_logging: 0.1603 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2730.9 | uss_MB: 1585.6 | child_rss_MB: 22461.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:21:03,552] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:21:03,623] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:21:47,382] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:21:47,382] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:49,878] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:22:49,878] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:54,751] INFO : Step 45000/500000 rank0 | Train Loss: 2.0273 | Train Loss Unweighted: 0.9366 | Train Loss Components: Classification: 0.0011 | Classification_binary: 0.7614 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0799 | Regression_eta: 0.0043 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0137 | Classification_binary: 0.7614 | Regression_cos_phi: 0.0357 | Regression_energy: 0.4561 | Regression_eta: 0.0275 | Regression_pt: 0.6871 | Regression_sin_phi: 0.0458 | LR: 1.19e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5689s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0366 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1757 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2731.2 | uss_MB: 1586.0 | child_rss_MB: 22462.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:23:52,095] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:23:52,095] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:45,006] INFO : Step 46000/500000 rank0 | Train Loss: 1.3640 | Train Loss Unweighted: 0.8152 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.7303 | Regression_cos_phi: 0.0097 | Regression_energy: 0.0324 | Regression_eta: 0.0038 | Regression_pt: 0.0359 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0065 | Classification_binary: 0.7303 | Regression_cos_phi: 0.0892 | Regression_energy: 0.1848 | Regression_eta: 0.0244 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0235 | LR: 1.23e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5132s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0394 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0711 | max_tensorboard_logging: 0.1285 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.8 | uss_MB: 1581.5 | child_rss_MB: 22461.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:24:45,008] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 05:24:45,080] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:24:54,333] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 05:24:54,333] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:56,157] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 05:25:56,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:35,484] INFO : Step 47000/500000 rank0 | Train Loss: 2.2112 | Train Loss Unweighted: 0.6465 | Train Loss Components: Classification: 0.0004 | Classification_binary: 0.3918 | Regression_cos_phi: 0.0029 | Regression_energy: 0.1223 | Regression_eta: 0.0041 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0055 | Classification_binary: 0.3918 | Regression_cos_phi: 0.0272 | Regression_energy: 0.6984 | Regression_eta: 0.0262 | Regression_pt: 1.0432 | Regression_sin_phi: 0.0188 | LR: 1.27e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5146s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0395 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1224 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2730.5 | uss_MB: 1585.3 | child_rss_MB: 22462.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:26:58,368] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 05:26:58,368] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:00,375] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 05:28:00,375] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:25,970] INFO : Step 48000/500000 rank0 | Train Loss: 2.6417 | Train Loss Unweighted: 1.4071 | Train Loss Components: Classification: 0.0009 | Classification_binary: 1.2068 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0938 | Regression_eta: 0.0060 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0114 | Classification_binary: 1.2068 | Regression_cos_phi: 0.0274 | Regression_energy: 0.5357 | Regression_eta: 0.0380 | Regression_pt: 0.7910 | Regression_sin_phi: 0.0313 | LR: 1.31e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5611s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0029 | max_backward: 0.0394 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1606 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.6 | uss_MB: 1585.3 | child_rss_MB: 22461.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:28:25,972] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 05:28:26,040] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:29:01,342] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:29:01,342] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:04,333] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:30:04,333] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:16,967] INFO : Step 49000/500000 rank0 | Train Loss: 0.9515 | Train Loss Unweighted: 0.5627 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.5009 | Regression_cos_phi: 0.0033 | Regression_energy: 0.0256 | Regression_eta: 0.0035 | Regression_pt: 0.0265 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0031 | Classification_binary: 0.5009 | Regression_cos_phi: 0.0306 | Regression_energy: 0.1462 | Regression_eta: 0.0221 | Regression_pt: 0.2254 | Regression_sin_phi: 0.0232 | LR: 1.36e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5881s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0387 | max_loss: 0.0031 | max_backward: 0.0408 | max_optimizer_step: 0.0700 | max_tensorboard_logging: 0.1742 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.6 | uss_MB: 1585.3 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 05:31:05,577] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:31:05,577] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:05,865] INFO : Step 50000/500000 rank0 | Train Loss: 1.7799 | Train Loss Unweighted: 0.7681 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.6068 | Regression_cos_phi: 0.0045 | Regression_energy: 0.0689 | Regression_eta: 0.0061 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 12.3659 | Classification_binary: 1.0000 | Regression_cos_phi: 9.2345 | Regression_energy: 5.7106 | Regression_eta: 6.3618 | Regression_pt: 8.5040 | Regression_sin_phi: 8.7625 | Task Weighted Losses: Classification: 0.0074 | Classification_binary: 0.6068 | Regression_cos_phi: 0.0420 | Regression_energy: 0.3933 | Regression_eta: 0.0386 | Regression_pt: 0.6446 | Regression_sin_phi: 0.0471 | LR: 1.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.6051s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0397 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.1900 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.4 | uss_MB: 1581.1 | child_rss_MB: 22460.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:32:05,867] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:32:05,940] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:32:05,941] INFO : Running validation on rank0 +rank0 [2026-11-14 05:32:05,941] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:32:05,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:32:06,062] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:32:06,170] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:32:06,271] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:32:06,340] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:32:06,408] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:32:06,477] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:32:06,542] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:32:06,606] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:32:06,718] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:32:06,827] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:32:06,930] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:32:06,998] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:32:07,065] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:32:07,130] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:32:07,198] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:32:07,266] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:32:07,368] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:32:07,374] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:32:07,376] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:32:07,378] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:32:07,380] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:32:07,381] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:32:07,383] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:32:07,384] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:32:07,385] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:32:07,387] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:32:07,389] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:32:07,391] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:32:07,393] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:32:07,395] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:32:07,397] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:32:07,399] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:32:08,815] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.7799 | Valid Loss=10.0218 | Train Loss Unweighted=0.7681 | Valid Loss Unweighted=6.4135 | Stale=3 | ETA=834.5m +rank0 [2026-11-14 05:32:08,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:32:09,394] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:32:09,403] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:09,457] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:32:09,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:32:10,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:32:10,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:32:10,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:32:10,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:32:11,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:32:11,409] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:32:11,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:32:11,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:32:12,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:32:12,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:32:12,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:32:12,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:32:13,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:32:13,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:32:13,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:32:13,614] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:32:13,621] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:14,167] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_2p5mm_seed12345_20261114_035850_139072/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:32:14,582] INFO : Plotted number of elements +rank0 [2026-11-14 05:32:15,866] INFO : Plotted jets +rank0 [2026-11-14 05:32:16,808] INFO : Plotted jet ratio +rank0 [2026-11-14 05:32:17,248] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:32:17,248] INFO : Training completed. Total time on device 0: 92.860min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..f6bcf6e86ad07f19aa76bfed1f5e04ea0f7f3316 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46ccf53dc780c71b3e0daba1b1da9b3ab1b6a8a50499a4c241fd33e36c3f6127 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-50000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..cba8957a448f77c7392d2147fb8c8a3f752614af --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02868478888c4b6049f5ff45091547d28afc79b85a4106739b9c0ec40a02f308 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..58becc0329c114a76c349d2638aa1db1f2ac7a1d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 2.375689744949341, "Classification": 0.026143040508031845, "Regression_pt": 0.09966954588890076, "Regression_eta": 0.010582922026515007, "Regression_sin_phi": 0.03039439395070076, "Regression_cos_phi": 0.019305363297462463, "Regression_energy": 0.10102790594100952, "Total": 4.70166540145874, "Total_unweighted": 2.662812916561961}, "train_group": {}, "valid": {"Classification_binary": 2.4731638431549072, "Classification": 0.01858219876885414, "Regression_pt": 0.3042246103286743, "Regression_eta": 0.01965334080159664, "Regression_sin_phi": 0.012752283364534378, "Regression_cos_phi": 0.011341908015310764, "Regression_energy": 0.3013206422328949, "Total": 7.562153339385986, "Total_unweighted": 3.1410388266667724, "metrics/particle/class_1/efficiency": 0.8682008385658264, "metrics/particle/class_1/pid_accuracy": 0.9445782899856567, "metrics/particle/class_2/efficiency": 0.2142857164144516, "metrics/particle/class_2/pid_accuracy": 0.8333333134651184, "metrics/particle/class_3/efficiency": 0.5783365368843079, "metrics/particle/class_3/pid_accuracy": 0.9799330830574036, "metrics/particle/class_4/efficiency": 0.8333333134651184, "metrics/particle/class_4/pid_accuracy": 0.9375, "metrics/particle/class_5/efficiency": 0.9518072009086609, "metrics/particle/class_5/pid_accuracy": 0.9620253443717957, "metrics/particle/count/bias_mean": -0.0560000017285347, "metrics/particle/count/mae": 0.5680000185966492, "metrics/particle/count/prediction_mean": 4.751999855041504, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.15073375403881073, "metrics/particle/event/energy_response_mean": 1.0424689054489136, "metrics/particle/event/met_abs_error": 4.84672737121582, "metrics/particle/event/scalar_pt_relative_abs_error": 0.1581195741891861, "metrics/particle/event/scalar_pt_response_mean": 1.0491948127746582, "metrics/particle/event/vector_pt_closure": 0.16692598164081573, "metrics/particle/matched/delta_eta_abs_mean": 0.02811259962618351, "metrics/particle/matched/delta_phi_abs_mean": 0.02462284080684185, "metrics/particle/matched/delta_r_mean": 0.04177157208323479, "metrics/particle/matched/energy_relative_abs_mean": 0.09211140125989914, "metrics/particle/matched/pid_accuracy": 0.9567690491676331, "metrics/particle/matched/pt_relative_abs_mean": 0.09087583422660828, "metrics/particle/matching/duplicate_fraction": 0.020202020183205605, "metrics/particle/matching/efficiency": 0.7312812209129333, "metrics/particle/matching/f1": 0.7355648279190063, "metrics/particle/matching/purity": 0.7398989796638489, "metrics/particle/matching/target_coverage_dr0p05": 0.5099833607673645, "metrics/particle/matching/target_coverage_dr0p10": 0.7703827023506165, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6930116415023804, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8311148285865784}, "step_train_time": 0.6915099620819092, "step_valid_time": 1.7631217109737918, "step_total_time": 718.6890687942505, "elapsed_training_loop_seconds": 718.6890687942505, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0141149086685675, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10366513361647, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 436, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 399, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 387, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8966292134831461, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9151376146788991, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.905788876276958, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08486238532110092, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8696629213483146, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8876146788990825, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8785471055618616, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8966292134831461, "step_test_time": 3.2255170400021598, "step_plot_time": 2.643612712970935, "step_validation_cycle_time": 8.119029654073529} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..3ee7b3fbec181c48f64a4a611fbaa31f3030fe00 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 2.5912153720855713, "Classification": 0.007965359836816788, "Regression_pt": 0.0782778412103653, "Regression_eta": 0.01464006956666708, "Regression_sin_phi": 0.009827327914536, "Regression_cos_phi": 0.010861181654036045, "Regression_energy": 0.08452502638101578, "Total": 4.170662879943848, "Total_unweighted": 2.7973121786490083}, "train_group": {}, "valid": {"Classification_binary": 2.4025750160217285, "Classification": 0.01700124330818653, "Regression_pt": 0.2776860296726227, "Regression_eta": 0.015979107469320297, "Regression_sin_phi": 0.009144416078925133, "Regression_cos_phi": 0.009956639260053635, "Regression_energy": 0.2768605053424835, "Total": 7.021111488342285, "Total_unweighted": 3.0092029571533203, "metrics/particle/class_1/efficiency": 0.8912134170532227, "metrics/particle/class_1/pid_accuracy": 0.9624413251876831, "metrics/particle/class_2/efficiency": 0.1428571492433548, "metrics/particle/class_2/pid_accuracy": 0.5, "metrics/particle/class_3/efficiency": 0.5899419784545898, "metrics/particle/class_3/pid_accuracy": 0.9770491719245911, "metrics/particle/class_4/efficiency": 0.8958333134651184, "metrics/particle/class_4/pid_accuracy": 0.9069767594337463, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9518072009086609, "metrics/particle/count/bias_mean": -0.10000000149011612, "metrics/particle/count/mae": 0.5320000052452087, "metrics/particle/count/prediction_mean": 4.708000183105469, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.10971105843782425, "metrics/particle/event/energy_response_mean": 0.9750187397003174, "metrics/particle/event/met_abs_error": 3.2365286350250244, "metrics/particle/event/scalar_pt_relative_abs_error": 0.11225485056638718, "metrics/particle/event/scalar_pt_response_mean": 0.9764251112937927, "metrics/particle/event/vector_pt_closure": 0.12339432537555695, "metrics/particle/matched/delta_eta_abs_mean": 0.02419067732989788, "metrics/particle/matched/delta_phi_abs_mean": 0.02104964479804039, "metrics/particle/matched/delta_r_mean": 0.035623762756586075, "metrics/particle/matched/energy_relative_abs_mean": 0.08320742845535278, "metrics/particle/matched/pid_accuracy": 0.9590708017349243, "metrics/particle/matched/pt_relative_abs_mean": 0.08214129507541656, "metrics/particle/matching/duplicate_fraction": 0.01614273525774479, "metrics/particle/matching/efficiency": 0.7520798444747925, "metrics/particle/matching/f1": 0.7599831819534302, "metrics/particle/matching/purity": 0.7680543661117554, "metrics/particle/matching/target_coverage_dr0p05": 0.5890182852745056, "metrics/particle/matching/target_coverage_dr0p10": 0.782029926776886, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7113144993782043, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8386023044586182}, "step_train_time": 0.7161123752593994, "step_valid_time": 1.884643139084801, "step_total_time": 1450.9323635101318, "elapsed_training_loop_seconds": 1450.9323635101318, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9898493453908901, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07698287216500721, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 387, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9078651685393259, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9308755760368663, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.919226393629124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06912442396313365, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8696629213483146, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8917050691244239, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8805460750853242, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9078651685393259, "step_test_time": 2.890479134977795, "step_plot_time": 2.1891867319354787, "step_validation_cycle_time": 7.462356757954694} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..6214057247b68eae5c6410232ee9ae9e31bf404f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 1.7726625204086304, "Classification": 0.004219860304147005, "Regression_pt": 0.04941563680768013, "Regression_eta": 0.012838931754231453, "Regression_sin_phi": 0.008958842605352402, "Regression_cos_phi": 0.005297640338540077, "Regression_energy": 0.05127621814608574, "Total": 2.7806241512298584, "Total_unweighted": 1.9046696503646672}, "train_group": {}, "valid": {"Classification_binary": 2.6860601902008057, "Classification": 0.019716257229447365, "Regression_pt": 0.3092282712459564, "Regression_eta": 0.017494050785899162, "Regression_sin_phi": 0.009005624800920486, "Regression_cos_phi": 0.009472272358834743, "Regression_energy": 0.3086550235748291, "Total": 7.814493179321289, "Total_unweighted": 3.359631690196693, "metrics/particle/class_1/efficiency": 0.8849372267723083, "metrics/particle/class_1/pid_accuracy": 0.9503546357154846, "metrics/particle/class_2/efficiency": 0.2142857164144516, "metrics/particle/class_2/pid_accuracy": 0.5, "metrics/particle/class_3/efficiency": 0.601547360420227, "metrics/particle/class_3/pid_accuracy": 0.9742765426635742, "metrics/particle/class_4/efficiency": 0.875, "metrics/particle/class_4/pid_accuracy": 0.9404761791229248, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9036144614219666, "metrics/particle/count/bias_mean": 0.02800000086426735, "metrics/particle/count/mae": 0.5640000104904175, "metrics/particle/count/prediction_mean": 4.835999965667725, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.10189796984195709, "metrics/particle/event/energy_response_mean": 0.9859110116958618, "metrics/particle/event/met_abs_error": 2.9942922592163086, "metrics/particle/event/scalar_pt_relative_abs_error": 0.10152340680360794, "metrics/particle/event/scalar_pt_response_mean": 0.9836679100990295, "metrics/particle/event/vector_pt_closure": 0.11338194459676743, "metrics/particle/matched/delta_eta_abs_mean": 0.02387121133506298, "metrics/particle/matched/delta_phi_abs_mean": 0.02072761207818985, "metrics/particle/matched/delta_r_mean": 0.03543776273727417, "metrics/particle/matched/energy_relative_abs_mean": 0.0829414427280426, "metrics/particle/matched/pid_accuracy": 0.9503858685493469, "metrics/particle/matched/pt_relative_abs_mean": 0.0836082249879837, "metrics/particle/matching/duplicate_fraction": 0.019023986533284187, "metrics/particle/matching/efficiency": 0.7545757293701172, "metrics/particle/matching/f1": 0.7523849010467529, "metrics/particle/matching/purity": 0.7502067685127258, "metrics/particle/matching/target_coverage_dr0p05": 0.5806988477706909, "metrics/particle/matching/target_coverage_dr0p10": 0.782029926776886, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7271214723587036, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8519134521484375}, "step_train_time": 0.7121286392211914, "step_valid_time": 1.9274540960323066, "step_total_time": 2187.021697998047, "elapsed_training_loop_seconds": 2187.021697998047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9885475429671627, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07809406192597301, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 413, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 401, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9280898876404494, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9407744874715261, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9343891402714932, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05922551252847381, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9011235955056179, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9134396355353075, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9072398190045249, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9280898876404494, "step_test_time": 3.060945500037633, "step_plot_time": 2.1792638229671866, "step_validation_cycle_time": 7.6245601259870455} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..16d23ec8cb8d55888f217b7cefea02bf32b2f595 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 0.2925780713558197, "Classification": 0.005151466466486454, "Regression_pt": 0.03494371473789215, "Regression_eta": 0.0046976059675216675, "Regression_sin_phi": 0.0026479766238480806, "Regression_cos_phi": 0.002885001478716731, "Regression_energy": 0.03816472366452217, "Total": 0.975779116153717, "Total_unweighted": 0.38106856029480696}, "train_group": {}, "valid": {"Classification_binary": 4.5177001953125, "Classification": 0.028040513396263123, "Regression_pt": 0.355609267950058, "Regression_eta": 0.017695574089884758, "Regression_sin_phi": 0.008770085871219635, "Regression_cos_phi": 0.010404546745121479, "Regression_energy": 0.3470292091369629, "Total": 10.400768280029297, "Total_unweighted": 5.28524939250201, "metrics/particle/class_1/efficiency": 0.8891213536262512, "metrics/particle/class_1/pid_accuracy": 0.9576470851898193, "metrics/particle/class_2/efficiency": 0.1428571492433548, "metrics/particle/class_2/pid_accuracy": 0.75, "metrics/particle/class_3/efficiency": 0.5841392874717712, "metrics/particle/class_3/pid_accuracy": 0.9768211841583252, "metrics/particle/class_4/efficiency": 0.9479166865348816, "metrics/particle/class_4/pid_accuracy": 0.8681318759918213, "metrics/particle/class_5/efficiency": 0.9879518151283264, "metrics/particle/class_5/pid_accuracy": 0.9390243887901306, "metrics/particle/count/bias_mean": 0.09600000083446503, "metrics/particle/count/mae": 0.5360000133514404, "metrics/particle/count/prediction_mean": 4.9039998054504395, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.11001171171665192, "metrics/particle/event/energy_response_mean": 1.0124754905700684, "metrics/particle/event/met_abs_error": 3.45824933052063, "metrics/particle/event/scalar_pt_relative_abs_error": 0.11184445768594742, "metrics/particle/event/scalar_pt_response_mean": 1.0137299299240112, "metrics/particle/event/vector_pt_closure": 0.1192539781332016, "metrics/particle/matched/delta_eta_abs_mean": 0.023595154285430908, "metrics/particle/matched/delta_phi_abs_mean": 0.020623449236154556, "metrics/particle/matched/delta_r_mean": 0.034918613731861115, "metrics/particle/matched/energy_relative_abs_mean": 0.08745001256465912, "metrics/particle/matched/pid_accuracy": 0.9524336457252502, "metrics/particle/matched/pt_relative_abs_mean": 0.08625203371047974, "metrics/particle/matching/duplicate_fraction": 0.022838499397039413, "metrics/particle/matching/efficiency": 0.7520798444747925, "metrics/particle/matching/f1": 0.7446457743644714, "metrics/particle/matching/purity": 0.7373572587966919, "metrics/particle/matching/target_coverage_dr0p05": 0.5915141701698303, "metrics/particle/matching/target_coverage_dr0p10": 0.7845258116722107, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7262895107269287, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8469218015670776}, "step_train_time": 0.7036349773406982, "step_valid_time": 1.913506795070134, "step_total_time": 2918.818705558777, "elapsed_training_loop_seconds": 2918.818705558777, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0185313857898972, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08761793496555026, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 394, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9078651685393259, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9308755760368663, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.919226393629124, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06912442396313365, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8853932584269663, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9078341013824884, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8964732650739476, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9078651685393259, "step_test_time": 2.9176415940746665, "step_plot_time": 2.1852146449964494, "step_validation_cycle_time": 7.487661138991825} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..4b467a7dc4db97e9a7f86c4f92b5aa6392f2d5f2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 0.7553850412368774, "Classification": 0.000284980284050107, "Regression_pt": 0.046845875680446625, "Regression_eta": 0.005573362112045288, "Regression_sin_phi": 0.0034983213990926743, "Regression_cos_phi": 0.002009709132835269, "Regression_energy": 0.04936712607741356, "Total": 1.5568513870239258, "Total_unweighted": 0.862964415922761}, "train_group": {}, "valid": {"Classification_binary": 5.638291358947754, "Classification": 0.03818804398179054, "Regression_pt": 0.34195882081985474, "Regression_eta": 0.017326179891824722, "Regression_sin_phi": 0.008424181491136551, "Regression_cos_phi": 0.008327445015311241, "Regression_energy": 0.3391270935535431, "Total": 11.454672813415527, "Total_unweighted": 6.391643123701215, "metrics/particle/class_1/efficiency": 0.9079498052597046, "metrics/particle/class_1/pid_accuracy": 0.9539170265197754, "metrics/particle/class_2/efficiency": 0.0714285746216774, "metrics/particle/class_2/pid_accuracy": 0.5, "metrics/particle/class_3/efficiency": 0.5725338459014893, "metrics/particle/class_3/pid_accuracy": 0.9594594836235046, "metrics/particle/class_4/efficiency": 0.90625, "metrics/particle/class_4/pid_accuracy": 0.8620689511299133, "metrics/particle/class_5/efficiency": 0.9879518151283264, "metrics/particle/class_5/pid_accuracy": 0.9390243887901306, "metrics/particle/count/bias_mean": 0.03200000151991844, "metrics/particle/count/mae": 0.527999997138977, "metrics/particle/count/prediction_mean": 4.840000152587891, "metrics/particle/count/target_mean": 4.808000087738037, "metrics/particle/event/energy_relative_abs_error": 0.11628234386444092, "metrics/particle/event/energy_response_mean": 1.004831075668335, "metrics/particle/event/met_abs_error": 3.7001852989196777, "metrics/particle/event/scalar_pt_relative_abs_error": 0.11864589154720306, "metrics/particle/event/scalar_pt_response_mean": 1.0170787572860718, "metrics/particle/event/vector_pt_closure": 0.12201939523220062, "metrics/particle/matched/delta_eta_abs_mean": 0.02624651789665222, "metrics/particle/matched/delta_phi_abs_mean": 0.018304014578461647, "metrics/particle/matched/delta_r_mean": 0.03547731786966324, "metrics/particle/matched/energy_relative_abs_mean": 0.07529155910015106, "metrics/particle/matched/pid_accuracy": 0.9445061087608337, "metrics/particle/matched/pt_relative_abs_mean": 0.07521902769804001, "metrics/particle/matching/duplicate_fraction": 0.01900826394557953, "metrics/particle/matching/efficiency": 0.7495840191841125, "metrics/particle/matching/f1": 0.7470978498458862, "metrics/particle/matching/purity": 0.7446280717849731, "metrics/particle/matching/target_coverage_dr0p05": 0.5815308094024658, "metrics/particle/matching/target_coverage_dr0p10": 0.7787021398544312, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.722129762172699, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8435940146446228}, "step_train_time": 0.7175302505493164, "step_valid_time": 1.869715568027459, "step_total_time": 3651.509374141693, "elapsed_training_loop_seconds": 3651.509374141693, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0141200954942549, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06935031092468136, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 445, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 439, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 410, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 399, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9213483146067416, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9339407744874715, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9276018099547512, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06605922551252848, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8966292134831461, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.908883826879271, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9027149321266968, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9213483146067416, "step_test_time": 2.896567676914856, "step_plot_time": 2.1770014599896967, "step_validation_cycle_time": 7.394854772952385} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..ac54e7deaa05002e8de46ecf13fbd933c391c126 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_z_tautau_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e70839f95c546b19eec0471f3a779215f37ee49a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66367d3d7760a95196b2894d532dec8b6038f530b135b2c88ea615cdf7ecfb44 +size 4402 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..be83ac21c1aa436b25d8118d52de0da556669cb9 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..051d795e9c3447deca813a61150b063e293d2f52 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ea7f69580b57824972b28d88aa6cc2957982e500 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..ce20e98abc8316f803c7fec7ca3bc913dc2117ca Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..27cf4989b6af5a1812b156e424c7c39ecfebdd51 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..f48c56cea2ab180d285aa658c781eca9ef9507fa Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ae02f3efa84b23e3bdcdf351d322844e24f9b4b8 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..533071cc82dd8cedc499775491849c470748f1c2 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7f61fe7ffea1aea45c878ec6432418eff6428457 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..14f77b079cf7e2df7c399991aab123b3e497226d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a38b0a07d0c7eb6e37be11952774b3b3a4fda3bf Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..b1d92317c7c7de6ee98941fea1cc9c6e9f195215 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/runs/train/events.out.tfevents.1794646874.workergpu215.3189551.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/runs/train/events.out.tfevents.1794646874.workergpu215.3189551.0 new file mode 100644 index 0000000000000000000000000000000000000000..fd2d9283dd8c43aacf3352596d076c57b2d54f90 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/runs/train/events.out.tfevents.1794646874.workergpu215.3189551.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3e80cf5ee4c938c978936101f69e96fd4d78f2e8318e2629eeb636cbd001e07 +size 1843423 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/runs/valid/events.out.tfevents.1794646875.workergpu215.3189551.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/runs/valid/events.out.tfevents.1794646875.workergpu215.3189551.1 new file mode 100644 index 0000000000000000000000000000000000000000..1a965b27de054f0bbc90e9d2f9c52ff76db21633 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/runs/valid/events.out.tfevents.1794646875.workergpu215.3189551.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f149ff600f9a4fdff362a5ed53a5db5835e042736c7101474af50acb1399009 +size 2257165 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..fea004f413eaf69f7471e2004b10fe59ec4b1f10 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_3mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e304f6f5edf7b102c55c0f1827d13765029ab4b5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_3mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/train.log new file mode 100644 index 0000000000000000000000000000000000000000..78063288b00c660dd1c5377820dd32a30292358a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/train.log @@ -0,0 +1,999 @@ +rank0 [2026-11-14 04:01:11,087] INFO : Will use single-gpu: NVIDIA H100 +rank0 [2026-11-14 04:01:11,087] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 04:01:12,418] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 04:01:12,418] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 04:01:12,419] INFO : Instantiating model +rank0 [2026-11-14 04:01:12,437] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 04:01:12,437] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 04:01:12,439] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 04:01:12,439] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 04:01:12,439] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 04:01:12,439] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 04:01:12,439] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,441] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 04:01:12,441] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,443] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 04:01:12,443] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,445] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 04:01:12,446] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,447] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 04:01:12,447] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,450] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 04:01:12,450] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,452] INFO : Backbone initialization took 0.01s +rank0 [2026-11-14 04:01:12,452] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 04:01:12,452] INFO : Initializing output DNNs +rank0 [2026-11-14 04:01:12,453] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,456] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,462] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 04:01:12,462] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 04:01:12,462] INFO : jagged_attention enabled=True +rank0 [2026-11-14 04:01:12,462] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 04:01:12,462] INFO : task_queries enabled=True +rank0 [2026-11-14 04:01:12,462] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 04:01:12,462] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 04:01:12,462] INFO : classification_query parameters: 256 +rank0 [2026-11-14 04:01:12,462] INFO : regression_query parameters: 256 +rank0 [2026-11-14 04:01:12,463] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 04:01:12,463] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 04:01:12,463] INFO : output_mode=elementwise +rank0 [2026-11-14 04:01:12,463] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 04:01:12,463] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 04:01:12,463] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 04:01:12,463] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 04:01:12,463] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 04:01:12,463] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 04:01:12,463] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 04:01:12,463] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 04:01:12,464] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 04:01:12,464] INFO : MLPF __init__ done +rank0 [2026-11-14 04:01:12,464] INFO : Instantiated model +rank0 [2026-11-14 04:01:12,464] INFO : Moving model to device rank=0 +rank0 [2026-11-14 04:01:12,470] INFO : Moved model to device rank=0 +rank0 [2026-11-14 04:01:12,471] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 04:01:12,537] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 04:01:12,537] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 04:01:12,538] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 04:01:12,538] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 04:01:12,538] INFO : Total parameters: 3910930 +rank0 [2026-11-14 04:01:12,539] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 04:01:12,541] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903 +rank0 [2026-11-14 04:01:12,541] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903 +rank0 [2026-11-14 04:01:12,541] INFO : Getting dataloaders +rank0 [2026-11-14 04:01:12,541] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:01:14,177] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,217] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,256] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,298] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,336] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,381] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,455] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,500] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,590] INFO : train_dataset: cld_edm_z_tautau_pf, 898 +rank0 [2026-11-14 04:01:14,659] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,659] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 04:01:14,659] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 04:01:14,659] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 04:01:14,659] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:01:14,669] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,679] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,689] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,699] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,709] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,718] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,728] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,738] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,747] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,757] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,757] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 04:01:14,757] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 04:01:14,792] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:01:14,994] INFO : Got dataloaders +rank0 [2026-11-14 04:01:14,994] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 04:01:14,994] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 04:01:14,995] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 04:01:15,003] INFO : Creating train_iterator +rank0 [2026-11-14 04:01:15,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:15,038] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:01:15,225] INFO : Created train_iterator +rank0 [2026-11-14 04:01:23,348] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.436795501708985, 'Classification': 0.598533673286438, 'Regression_pt': 0.8346810086071491, 'Regression_eta': 1.176606583595276, 'Regression_sin_phi': 0.8390423528850078, 'Regression_cos_phi': 0.8456102706491947, 'Regression_energy': 1.2376715517044068}, weights={'Classification_binary': 1.0, 'Classification': 12.425024032592773, 'Regression_pt': 8.909745216369629, 'Regression_eta': 6.320545673370361, 'Regression_sin_phi': 8.863432884216309, 'Regression_cos_phi': 8.79458999633789, 'Regression_energy': 6.0086989402771} +rank0 [2026-11-14 04:01:56,726] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 04:01:56,726] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:02:28,485] INFO : Step 1000/500000 rank0 | Train Loss: 10.8482 | Train Loss Unweighted: 5.2339 | Train Loss Components: Classification: 0.0953 | Classification_binary: 4.4474 | Regression_cos_phi: 0.0295 | Regression_energy: 0.2843 | Regression_eta: 0.0435 | Regression_pt: 0.3069 | Regression_sin_phi: 0.0271 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 1.1841 | Classification_binary: 4.4474 | Regression_cos_phi: 0.2596 | Regression_energy: 1.7080 | Regression_eta: 0.2752 | Regression_pt: 2.7340 | Regression_sin_phi: 0.2399 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7171s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0250 | max_optimizer_step: 0.0780 | max_tensorboard_logging: 0.2286 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2589.8 | uss_MB: 1340.5 | child_rss_MB: 22726.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:02:37,106] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:02:37,106] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:16,563] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:03:16,563] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:39,547] INFO : Step 2000/500000 rank0 | Train Loss: 8.6441 | Train Loss Unweighted: 3.4456 | Train Loss Components: Classification: 0.0401 | Classification_binary: 2.6864 | Regression_cos_phi: 0.0118 | Regression_energy: 0.3166 | Regression_eta: 0.0105 | Regression_pt: 0.3674 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.4982 | Classification_binary: 2.6864 | Regression_cos_phi: 0.1039 | Regression_energy: 1.9025 | Regression_eta: 0.0661 | Regression_pt: 3.2738 | Regression_sin_phi: 0.1132 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.6989s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0247 | max_optimizer_step: 0.0762 | max_tensorboard_logging: 0.2160 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2590.5 | uss_MB: 1340.4 | child_rss_MB: 22722.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:03:39,549] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:03:56,665] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:03:56,665] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:36,999] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:04:36,999] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:50,856] INFO : Step 3000/500000 rank0 | Train Loss: 13.6524 | Train Loss Unweighted: 5.7479 | Train Loss Components: Classification: 0.0648 | Classification_binary: 4.5797 | Regression_cos_phi: 0.0296 | Regression_energy: 0.5128 | Regression_eta: 0.0275 | Regression_pt: 0.5178 | Regression_sin_phi: 0.0158 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.8046 | Classification_binary: 4.5797 | Regression_cos_phi: 0.2601 | Regression_energy: 3.0812 | Regression_eta: 0.1738 | Regression_pt: 4.6132 | Regression_sin_phi: 0.1399 | LR: 2.05e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.6969s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0762 | max_tensorboard_logging: 0.2176 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2622.4 | uss_MB: 1372.6 | child_rss_MB: 22722.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:05:17,262] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:05:17,262] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:56,746] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:05:56,747] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:02,045] INFO : Step 4000/500000 rank0 | Train Loss: 4.7707 | Train Loss Unweighted: 3.1381 | Train Loss Components: Classification: 0.0355 | Classification_binary: 2.9134 | Regression_cos_phi: 0.0075 | Regression_energy: 0.0836 | Regression_eta: 0.0100 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.4412 | Classification_binary: 2.9134 | Regression_cos_phi: 0.0659 | Regression_energy: 0.5024 | Regression_eta: 0.0629 | Regression_pt: 0.7231 | Regression_sin_phi: 0.0619 | LR: 2.08e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6936s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0231 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0764 | max_tensorboard_logging: 0.2148 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2622.4 | uss_MB: 1372.6 | child_rss_MB: 22722.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:06:02,049] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:06:37,344] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:06:37,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:13,913] INFO : Step 5000/500000 rank0 | Train Loss: 19.4153 | Train Loss Unweighted: 5.5590 | Train Loss Components: Classification: 0.0360 | Classification_binary: 3.4586 | Regression_cos_phi: 0.0315 | Regression_energy: 0.9579 | Regression_eta: 0.0370 | Regression_pt: 0.9176 | Regression_sin_phi: 0.1204 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.4479 | Classification_binary: 3.4586 | Regression_cos_phi: 0.2769 | Regression_energy: 5.7557 | Regression_eta: 0.2338 | Regression_pt: 8.1755 | Regression_sin_phi: 1.0670 | LR: 2.13e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7008s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0230 | max_loss: 0.0022 | max_backward: 0.0255 | max_optimizer_step: 0.0760 | max_tensorboard_logging: 0.2215 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2592.6 | uss_MB: 1342.8 | child_rss_MB: 22722.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:07:18,002] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:07:18,002] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:57,754] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:07:57,754] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:25,373] INFO : Step 6000/500000 rank0 | Train Loss: 9.5847 | Train Loss Unweighted: 4.5101 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.7352 | Regression_cos_phi: 0.0278 | Regression_energy: 0.3257 | Regression_eta: 0.0603 | Regression_pt: 0.3043 | Regression_sin_phi: 0.0424 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1803 | Classification_binary: 3.7352 | Regression_cos_phi: 0.2441 | Regression_energy: 1.9570 | Regression_eta: 0.3809 | Regression_pt: 2.7112 | Regression_sin_phi: 0.3759 | LR: 2.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.6980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0231 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0778 | max_tensorboard_logging: 0.2128 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2592.7 | uss_MB: 1342.9 | child_rss_MB: 22723.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:08:25,375] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:08:38,325] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:08:38,325] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:18,273] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:09:18,273] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:36,719] INFO : Step 7000/500000 rank0 | Train Loss: 4.2768 | Train Loss Unweighted: 2.2646 | Train Loss Components: Classification: 0.0196 | Classification_binary: 1.9713 | Regression_cos_phi: 0.0037 | Regression_energy: 0.1205 | Regression_eta: 0.0100 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.2433 | Classification_binary: 1.9713 | Regression_cos_phi: 0.0324 | Regression_energy: 0.7238 | Regression_eta: 0.0632 | Regression_pt: 1.1511 | Regression_sin_phi: 0.0917 | LR: 2.26e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7031s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0249 | max_optimizer_step: 0.0759 | max_tensorboard_logging: 0.2178 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2593.3 | uss_MB: 1343.5 | child_rss_MB: 22722.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:09:58,242] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:09:58,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:37,978] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:10:37,978] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:47,944] INFO : Step 8000/500000 rank0 | Train Loss: 8.6984 | Train Loss Unweighted: 3.1007 | Train Loss Components: Classification: 0.0496 | Classification_binary: 2.2744 | Regression_cos_phi: 0.0114 | Regression_energy: 0.3773 | Regression_eta: 0.0063 | Regression_pt: 0.3770 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.6157 | Classification_binary: 2.2744 | Regression_cos_phi: 0.1006 | Regression_energy: 2.2669 | Regression_eta: 0.0396 | Regression_pt: 3.3590 | Regression_sin_phi: 0.0422 | LR: 2.34e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6902s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0256 | max_optimizer_step: 0.0781 | max_tensorboard_logging: 0.2098 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2593.3 | uss_MB: 1343.5 | child_rss_MB: 22721.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:10:47,946] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:10:48,006] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:11:18,552] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:11:18,552] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:58,615] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:11:58,615] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:59,834] INFO : Step 9000/500000 rank0 | Train Loss: 5.1573 | Train Loss Unweighted: 2.1450 | Train Loss Components: Classification: 0.0071 | Classification_binary: 1.6877 | Regression_cos_phi: 0.0055 | Regression_energy: 0.2048 | Regression_eta: 0.0133 | Regression_pt: 0.2224 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 1.6877 | Regression_cos_phi: 0.0481 | Regression_energy: 1.2305 | Regression_eta: 0.0838 | Regression_pt: 1.9814 | Regression_sin_phi: 0.0371 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.6944s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0231 | max_loss: 0.0022 | max_backward: 0.0257 | max_optimizer_step: 0.0765 | max_tensorboard_logging: 0.2138 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2591.3 | uss_MB: 1341.6 | child_rss_MB: 22724.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:12:39,251] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:12:39,251] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:11,869] INFO : Step 10000/500000 rank0 | Train Loss: 4.7017 | Train Loss Unweighted: 2.6628 | Train Loss Components: Classification: 0.0261 | Classification_binary: 2.3757 | Regression_cos_phi: 0.0193 | Regression_energy: 0.1010 | Regression_eta: 0.0106 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0304 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.3248 | Classification_binary: 2.3757 | Regression_cos_phi: 0.1698 | Regression_energy: 0.6070 | Regression_eta: 0.0669 | Regression_pt: 0.8880 | Regression_sin_phi: 0.2694 | LR: 2.52e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6904s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0230 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.2115 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2591.9 | uss_MB: 1342.0 | child_rss_MB: 22724.7 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:13:11,871] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:13:11,920] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:13:11,921] INFO : Running validation on rank0 +rank0 [2026-11-14 04:13:11,921] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:13:11,921] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:13:12,052] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:13:12,183] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:13:12,232] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:13:12,270] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:13:12,309] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:13:12,342] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:13:12,381] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:13:12,419] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:13:12,453] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:13:12,499] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:13:12,545] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:13:12,583] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:13:12,622] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:13:12,655] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:13:12,693] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:13:12,732] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:13:12,768] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:13:12,770] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:13:12,771] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:13:12,773] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:13:12,774] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:13:12,776] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:13:12,777] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:13:12,779] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:13:12,780] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:13:12,782] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:13:12,783] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:13:12,785] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:13:12,786] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:13:12,786] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:13:12,788] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:13:12,789] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:13:13,742] INFO : VALIDATION | Step=10000/500000 | Train Loss=4.7017 | Valid Loss=7.5622 | Train Loss Unweighted=2.6628 | Valid Loss Unweighted=3.1410 | Stale=0 | ETA=587.0m +rank0 [2026-11-14 04:13:13,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:13:13,840] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:13:13,849] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:13,902] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:13:14,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:13:14,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:13:14,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:13:15,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:13:15,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:13:15,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:13:15,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:13:15,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:13:15,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:13:15,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:13:16,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:13:16,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:13:16,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:13:16,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:13:16,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:13:16,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:13:16,966] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 04:13:16,973] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:17,325] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:18,030] INFO : Plotted number of elements +rank0 [2026-11-14 04:13:18,941] INFO : Plotted jets +rank0 [2026-11-14 04:13:19,605] INFO : Plotted jet ratio +rank0 [2026-11-14 04:13:28,360] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:13:28,360] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:08,684] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:14:08,684] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:32,491] INFO : Step 11000/500000 rank0 | Train Loss: 7.1059 | Train Loss Unweighted: 2.3937 | Train Loss Components: Classification: 0.0049 | Classification_binary: 1.6778 | Regression_cos_phi: 0.0192 | Regression_energy: 0.3223 | Regression_eta: 0.0114 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0249 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 1.6778 | Regression_cos_phi: 0.1690 | Regression_energy: 1.9366 | Regression_eta: 0.0724 | Regression_pt: 2.9685 | Regression_sin_phi: 0.2210 | LR: 2.63e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7242s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0262 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2155 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.1 | uss_MB: 1483.6 | child_rss_MB: 22724.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:14:49,751] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:14:49,751] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:30,083] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:15:30,084] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:44,917] INFO : Step 12000/500000 rank0 | Train Loss: 3.0135 | Train Loss Unweighted: 1.8501 | Train Loss Components: Classification: 0.0091 | Classification_binary: 1.6765 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0728 | Regression_eta: 0.0116 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1132 | Classification_binary: 1.6765 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4377 | Regression_eta: 0.0732 | Regression_pt: 0.6259 | Regression_sin_phi: 0.0393 | LR: 2.75e-05 | DataLoad Time: 0.0031s | Model Forward Time: 0.7123s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0263 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2119 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.2 | uss_MB: 1483.7 | child_rss_MB: 22723.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:15:44,919] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:15:44,967] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:16:11,028] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:16:11,028] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:16:51,730] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:16:51,730] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:16:57,717] INFO : Step 13000/500000 rank0 | Train Loss: 8.9040 | Train Loss Unweighted: 3.5480 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.7349 | Regression_cos_phi: 0.0114 | Regression_energy: 0.3668 | Regression_eta: 0.0245 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1891 | Classification_binary: 2.7349 | Regression_cos_phi: 0.1001 | Regression_energy: 2.2037 | Regression_eta: 0.1548 | Regression_pt: 3.3933 | Regression_sin_phi: 0.1281 | LR: 2.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7107s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0776 | max_tensorboard_logging: 0.2142 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2671.4 | uss_MB: 1428.9 | child_rss_MB: 22722.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:17:32,786] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:17:32,786] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:10,207] INFO : Step 14000/500000 rank0 | Train Loss: 3.4496 | Train Loss Unweighted: 1.9632 | Train Loss Components: Classification: 0.0042 | Classification_binary: 1.7365 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1025 | Regression_eta: 0.0090 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 1.7365 | Regression_cos_phi: 0.0604 | Regression_energy: 0.6158 | Regression_eta: 0.0568 | Regression_pt: 0.8793 | Regression_sin_phi: 0.0487 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7052s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2671.9 | uss_MB: 1429.5 | child_rss_MB: 22722.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:18:10,208] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:18:10,256] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:18:13,874] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:18:13,874] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:54,357] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:18:54,357] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:23,207] INFO : Step 15000/500000 rank0 | Train Loss: 6.1661 | Train Loss Unweighted: 2.7591 | Train Loss Components: Classification: 0.0108 | Classification_binary: 2.2285 | Regression_cos_phi: 0.0045 | Regression_energy: 0.2574 | Regression_eta: 0.0306 | Regression_pt: 0.2118 | Regression_sin_phi: 0.0154 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1339 | Classification_binary: 2.2285 | Regression_cos_phi: 0.0399 | Regression_energy: 1.5465 | Regression_eta: 0.1936 | Regression_pt: 1.8871 | Regression_sin_phi: 0.1365 | LR: 3.17e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7072s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2128 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2675.3 | uss_MB: 1432.9 | child_rss_MB: 22723.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:19:35,681] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:19:35,682] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:16,141] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:20:16,141] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:35,746] INFO : Step 16000/500000 rank0 | Train Loss: 9.1400 | Train Loss Unweighted: 2.9519 | Train Loss Components: Classification: 0.0127 | Classification_binary: 1.9928 | Regression_cos_phi: 0.0133 | Regression_energy: 0.4415 | Regression_eta: 0.0615 | Regression_pt: 0.3999 | Regression_sin_phi: 0.0301 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1584 | Classification_binary: 1.9928 | Regression_cos_phi: 0.1170 | Regression_energy: 2.6529 | Regression_eta: 0.3889 | Regression_pt: 3.5630 | Regression_sin_phi: 0.2670 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7074s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.2146 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2675.3 | uss_MB: 1432.9 | child_rss_MB: 22723.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:20:35,748] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:20:35,796] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:20:57,212] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:20:57,212] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:37,641] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:21:37,641] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:48,178] INFO : Step 17000/500000 rank0 | Train Loss: 3.0402 | Train Loss Unweighted: 1.8028 | Train Loss Components: Classification: 0.0132 | Classification_binary: 1.6237 | Regression_cos_phi: 0.0184 | Regression_energy: 0.0646 | Regression_eta: 0.0137 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1645 | Classification_binary: 1.6237 | Regression_cos_phi: 0.1616 | Regression_energy: 0.3879 | Regression_eta: 0.0867 | Regression_pt: 0.5304 | Regression_sin_phi: 0.0853 | LR: 3.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7020s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0782 | max_tensorboard_logging: 0.2080 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2671.3 | uss_MB: 1428.9 | child_rss_MB: 22724.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:22:18,870] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:22:18,871] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:59,165] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:22:59,165] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:00,904] INFO : Step 18000/500000 rank0 | Train Loss: 4.1121 | Train Loss Unweighted: 2.4614 | Train Loss Components: Classification: 0.0067 | Classification_binary: 2.2116 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1136 | Regression_eta: 0.0070 | Regression_pt: 0.1103 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0833 | Classification_binary: 2.2116 | Regression_cos_phi: 0.0615 | Regression_energy: 0.6829 | Regression_eta: 0.0440 | Regression_pt: 0.9827 | Regression_sin_phi: 0.0462 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.6940s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0233 | max_loss: 0.0022 | max_backward: 0.0250 | max_optimizer_step: 0.0783 | max_tensorboard_logging: 0.1987 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2672.1 | uss_MB: 1429.6 | child_rss_MB: 22723.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:23:00,906] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:23:00,953] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:23:39,328] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:23:39,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:12,310] INFO : Step 19000/500000 rank0 | Train Loss: 4.0787 | Train Loss Unweighted: 2.2921 | Train Loss Components: Classification: 0.0068 | Classification_binary: 2.0221 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1199 | Regression_eta: 0.0093 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0846 | Classification_binary: 2.0221 | Regression_cos_phi: 0.0296 | Regression_energy: 0.7207 | Regression_eta: 0.0586 | Regression_pt: 1.0965 | Regression_sin_phi: 0.0665 | LR: 3.88e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7337s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0291 | max_loss: 0.0022 | max_backward: 0.0254 | max_optimizer_step: 0.0785 | max_tensorboard_logging: 0.2023 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2676.1 | uss_MB: 1433.7 | child_rss_MB: 22724.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:19,908] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:24:19,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:59,255] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:24:59,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:23,999] INFO : Step 20000/500000 rank0 | Train Loss: 4.1707 | Train Loss Unweighted: 2.7973 | Train Loss Components: Classification: 0.0080 | Classification_binary: 2.5912 | Regression_cos_phi: 0.0109 | Regression_energy: 0.0845 | Regression_eta: 0.0146 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0990 | Classification_binary: 2.5912 | Regression_cos_phi: 0.0955 | Regression_energy: 0.5079 | Regression_eta: 0.0925 | Regression_pt: 0.6974 | Regression_sin_phi: 0.0871 | LR: 4.07e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7150s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0795 | max_tensorboard_logging: 0.2116 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2677.0 | uss_MB: 1434.6 | child_rss_MB: 22724.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:25:24,001] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:25:24,048] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:25:24,049] INFO : Running validation on rank0 +rank0 [2026-11-14 04:25:24,049] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:25:24,050] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:25:24,128] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:25:24,201] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:25:24,270] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:25:24,317] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:25:24,363] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:25:24,409] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:25:24,455] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:25:24,500] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:25:24,569] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:25:24,637] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:25:24,703] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:25:24,749] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:25:24,794] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:25:24,838] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:25:24,883] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:25:24,927] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:25:24,993] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:25:24,997] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:25:24,999] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:25:25,000] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:25:25,002] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:25:25,003] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:25:25,005] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:25:25,006] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:25:25,008] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:25:25,009] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:25:25,009] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:25:25,011] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:25:25,012] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:25:25,013] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:25:25,014] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:25:25,016] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:25:25,984] INFO : VALIDATION | Step=20000/500000 | Train Loss=4.1707 | Valid Loss=7.0211 | Train Loss Unweighted=2.7973 | Valid Loss Unweighted=3.0092 | Stale=0 | ETA=580.4m +rank0 [2026-11-14 04:25:25,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:25:26,076] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:25:26,085] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:26,128] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:25:26,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:25:26,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:25:26,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:25:26,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:25:27,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:25:27,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:25:27,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:25:27,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:25:27,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:25:27,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:25:28,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:25:28,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:25:28,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:25:28,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:25:28,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:25:28,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:25:28,873] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:25:28,879] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:29,232] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:29,504] INFO : Plotted number of elements +rank0 [2026-11-14 04:25:30,382] INFO : Plotted jets +rank0 [2026-11-14 04:25:31,059] INFO : Plotted jet ratio +rank0 [2026-11-14 04:25:48,266] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:25:48,266] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:28,637] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:26:28,637] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:44,150] INFO : Step 21000/500000 rank0 | Train Loss: 7.6543 | Train Loss Unweighted: 3.7666 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.1977 | Regression_cos_phi: 0.1132 | Regression_energy: 0.2088 | Regression_eta: 0.0151 | Regression_pt: 0.2125 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1625 | Classification_binary: 3.1977 | Regression_cos_phi: 0.9955 | Regression_energy: 1.2544 | Regression_eta: 0.0957 | Regression_pt: 1.8936 | Regression_sin_phi: 0.0549 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0234 | max_loss: 0.0022 | max_backward: 0.0260 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2297 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.3 | uss_MB: 1461.9 | child_rss_MB: 22725.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:27:09,819] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:27:09,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:50,277] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:27:50,278] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:56,856] INFO : Step 22000/500000 rank0 | Train Loss: 2.8379 | Train Loss Unweighted: 1.6083 | Train Loss Components: Classification: 0.0094 | Classification_binary: 1.4284 | Regression_cos_phi: 0.0055 | Regression_energy: 0.0743 | Regression_eta: 0.0039 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1171 | Classification_binary: 1.4284 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4465 | Regression_eta: 0.0246 | Regression_pt: 0.7404 | Regression_sin_phi: 0.0323 | LR: 4.50e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7176s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.0806 | max_tensorboard_logging: 0.2118 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2705.3 | uss_MB: 1462.9 | child_rss_MB: 22723.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:27:56,858] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:27:56,905] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:31,802] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:28:31,802] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:09,962] INFO : Step 23000/500000 rank0 | Train Loss: 4.9195 | Train Loss Unweighted: 2.3961 | Train Loss Components: Classification: 0.0083 | Classification_binary: 2.0136 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1783 | Regression_eta: 0.0052 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1033 | Classification_binary: 2.0136 | Regression_cos_phi: 0.0400 | Regression_energy: 1.0716 | Regression_eta: 0.0331 | Regression_pt: 1.6130 | Regression_sin_phi: 0.0450 | LR: 4.73e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7181s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0784 | max_tensorboard_logging: 0.2173 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2708.7 | uss_MB: 1466.3 | child_rss_MB: 22723.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:29:12,997] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:29:12,998] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:53,645] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:29:53,645] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:22,880] INFO : Step 24000/500000 rank0 | Train Loss: 3.5262 | Train Loss Unweighted: 2.3697 | Train Loss Components: Classification: 0.0108 | Classification_binary: 2.2038 | Regression_cos_phi: 0.0091 | Regression_energy: 0.0591 | Regression_eta: 0.0081 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1347 | Classification_binary: 2.2038 | Regression_cos_phi: 0.0803 | Regression_energy: 0.3553 | Regression_eta: 0.0510 | Regression_pt: 0.6453 | Regression_sin_phi: 0.0557 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7126s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0233 | max_loss: 0.0022 | max_backward: 0.0260 | max_optimizer_step: 0.0786 | max_tensorboard_logging: 0.2142 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2708.7 | uss_MB: 1466.3 | child_rss_MB: 22723.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:30:22,881] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:30:22,929] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:30:34,919] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:30:34,919] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:15,469] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:31:15,469] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:36,128] INFO : Step 25000/500000 rank0 | Train Loss: 2.9109 | Train Loss Unweighted: 1.8483 | Train Loss Components: Classification: 0.0086 | Classification_binary: 1.6940 | Regression_cos_phi: 0.0139 | Regression_energy: 0.0599 | Regression_eta: 0.0047 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1067 | Classification_binary: 1.6940 | Regression_cos_phi: 0.1225 | Regression_energy: 0.3599 | Regression_eta: 0.0295 | Regression_pt: 0.5461 | Regression_sin_phi: 0.0523 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7057s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0236 | max_loss: 0.0022 | max_backward: 0.0262 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.2088 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.3 | uss_MB: 1461.9 | child_rss_MB: 22722.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:31:57,038] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:31:57,038] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:37,547] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:32:37,547] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:48,901] INFO : Step 26000/500000 rank0 | Train Loss: 1.9940 | Train Loss Unweighted: 1.3781 | Train Loss Components: Classification: 0.0013 | Classification_binary: 1.2876 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0290 | Regression_eta: 0.0078 | Regression_pt: 0.0387 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0164 | Classification_binary: 1.2876 | Regression_cos_phi: 0.0246 | Regression_energy: 0.1745 | Regression_eta: 0.0494 | Regression_pt: 0.3452 | Regression_sin_phi: 0.0963 | LR: 5.47e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2206 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.8 | uss_MB: 1462.4 | child_rss_MB: 22724.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:32:48,903] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:32:48,952] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:33:18,782] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:33:18,782] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:59,313] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:33:59,313] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:01,721] INFO : Step 27000/500000 rank0 | Train Loss: 2.9179 | Train Loss Unweighted: 1.2410 | Train Loss Components: Classification: 0.0051 | Classification_binary: 0.9867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.1180 | Regression_eta: 0.0038 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.9867 | Regression_cos_phi: 0.0226 | Regression_energy: 0.7092 | Regression_eta: 0.0237 | Regression_pt: 1.0885 | Regression_sin_phi: 0.0234 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7036s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0241 | max_loss: 0.0022 | max_backward: 0.0264 | max_optimizer_step: 0.0774 | max_tensorboard_logging: 0.2095 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2708.8 | uss_MB: 1466.4 | child_rss_MB: 22724.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:34:40,798] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:34:40,798] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:14,695] INFO : Step 28000/500000 rank0 | Train Loss: 4.9368 | Train Loss Unweighted: 2.0786 | Train Loss Components: Classification: 0.0261 | Classification_binary: 1.6569 | Regression_cos_phi: 0.0027 | Regression_energy: 0.1906 | Regression_eta: 0.0062 | Regression_pt: 0.1921 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.3239 | Classification_binary: 1.6569 | Regression_cos_phi: 0.0238 | Regression_energy: 1.1451 | Regression_eta: 0.0393 | Regression_pt: 1.7118 | Regression_sin_phi: 0.0359 | LR: 6.01e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7113s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0278 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2119 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2709.0 | uss_MB: 1466.6 | child_rss_MB: 22726.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:14,697] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:35:14,743] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:35:21,953] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:35:21,953] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:02,465] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:36:02,465] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:27,494] INFO : Step 29000/500000 rank0 | Train Loss: 3.8558 | Train Loss Unweighted: 1.7969 | Train Loss Components: Classification: 0.0061 | Classification_binary: 1.4840 | Regression_cos_phi: 0.0041 | Regression_energy: 0.1465 | Regression_eta: 0.0047 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 1.4840 | Regression_cos_phi: 0.0358 | Regression_energy: 0.8801 | Regression_eta: 0.0299 | Regression_pt: 1.3049 | Regression_sin_phi: 0.0452 | LR: 6.29e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7062s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0022 | max_backward: 0.0260 | max_optimizer_step: 0.0784 | max_tensorboard_logging: 0.2078 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2709.8 | uss_MB: 1467.4 | child_rss_MB: 22725.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:36:43,488] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:36:43,488] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:23,683] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:37:23,683] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:40,045] INFO : Step 30000/500000 rank0 | Train Loss: 2.7806 | Train Loss Unweighted: 1.9047 | Train Loss Components: Classification: 0.0042 | Classification_binary: 1.7727 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0513 | Regression_eta: 0.0128 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 1.7727 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3081 | Regression_eta: 0.0811 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0794 | LR: 6.58e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7115s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0251 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2112 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.9 | uss_MB: 1462.4 | child_rss_MB: 22725.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:37:40,047] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:37:40,096] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:37:40,096] INFO : Running validation on rank0 +rank0 [2026-11-14 04:37:40,096] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:37:40,096] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:37:40,203] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:37:40,277] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:37:40,346] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:37:40,394] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:37:40,440] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:37:40,486] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:37:40,532] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:37:40,579] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:37:40,653] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:37:40,721] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:37:40,789] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:37:40,835] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:37:40,881] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:37:40,926] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:37:40,971] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:37:41,017] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:37:41,089] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:37:41,093] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:37:41,095] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:37:41,097] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:37:41,098] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:37:41,100] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:37:41,101] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:37:41,103] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:37:41,104] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:37:41,106] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:37:41,107] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:37:41,108] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:37:41,110] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:37:41,110] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:37:41,112] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:37:41,113] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:37:42,028] INFO : VALIDATION | Step=30000/500000 | Train Loss=2.7806 | Valid Loss=7.8145 | Train Loss Unweighted=1.9047 | Valid Loss Unweighted=3.3596 | Stale=1 | ETA=571.1m +rank0 [2026-11-14 04:37:42,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:37:42,137] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:37:42,146] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:37:42,207] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:37:42,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:37:42,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:37:43,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:37:43,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:37:43,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:37:43,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:37:43,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:37:43,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:37:43,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:37:44,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:37:44,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:37:44,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:37:44,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:37:44,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:37:44,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:37:44,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:37:45,087] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:37:45,094] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:37:45,452] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:37:45,710] INFO : Plotted number of elements +rank0 [2026-11-14 04:37:46,576] INFO : Plotted jets +rank0 [2026-11-14 04:37:47,263] INFO : Plotted jet ratio +rank0 [2026-11-14 04:38:12,405] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:38:12,405] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:38:53,024] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:38:53,024] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:00,091] INFO : Step 31000/500000 rank0 | Train Loss: 1.4948 | Train Loss Unweighted: 0.8320 | Train Loss Components: Classification: 0.0072 | Classification_binary: 0.7342 | Regression_cos_phi: 0.0056 | Regression_energy: 0.0381 | Regression_eta: 0.0097 | Regression_pt: 0.0338 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0898 | Classification_binary: 0.7342 | Regression_cos_phi: 0.0493 | Regression_energy: 0.2287 | Regression_eta: 0.0613 | Regression_pt: 0.3014 | Regression_sin_phi: 0.0301 | LR: 6.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7328s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0793 | max_tensorboard_logging: 0.2171 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2822.7 | uss_MB: 1580.3 | child_rss_MB: 22724.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:39:34,137] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:39:34,137] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:13,014] INFO : Step 32000/500000 rank0 | Train Loss: 2.0666 | Train Loss Unweighted: 0.9571 | Train Loss Components: Classification: 0.0099 | Classification_binary: 0.7926 | Regression_cos_phi: 0.0077 | Regression_energy: 0.0687 | Regression_eta: 0.0101 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1230 | Classification_binary: 0.7926 | Regression_cos_phi: 0.0678 | Regression_energy: 0.4126 | Regression_eta: 0.0639 | Regression_pt: 0.5675 | Regression_sin_phi: 0.0392 | LR: 7.19e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7122s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2064 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2822.7 | uss_MB: 1580.3 | child_rss_MB: 22724.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:40:13,016] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:40:13,063] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:40:15,525] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:40:15,525] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:56,324] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:40:56,324] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:25,583] INFO : Step 33000/500000 rank0 | Train Loss: 1.5106 | Train Loss Unweighted: 1.0012 | Train Loss Components: Classification: 0.0022 | Classification_binary: 0.9252 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0301 | Regression_eta: 0.0046 | Regression_pt: 0.0310 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0277 | Classification_binary: 0.9252 | Regression_cos_phi: 0.0282 | Regression_energy: 0.1807 | Regression_eta: 0.0289 | Regression_pt: 0.2761 | Regression_sin_phi: 0.0439 | LR: 7.51e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7123s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0251 | max_optimizer_step: 0.0777 | max_tensorboard_logging: 0.2193 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2728.9 | uss_MB: 1486.5 | child_rss_MB: 22723.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:41:36,777] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 04:41:36,777] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:17,344] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 04:42:17,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:38,205] INFO : Step 34000/500000 rank0 | Train Loss: 1.8048 | Train Loss Unweighted: 1.2016 | Train Loss Components: Classification: 0.0016 | Classification_binary: 1.1100 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0370 | Regression_eta: 0.0076 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0195 | Classification_binary: 1.1100 | Regression_cos_phi: 0.0349 | Regression_energy: 0.2221 | Regression_eta: 0.0479 | Regression_pt: 0.3467 | Regression_sin_phi: 0.0238 | LR: 7.83e-05 | DataLoad Time: 0.0136s | Model Forward Time: 0.6943s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2005 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2729.4 | uss_MB: 1487.0 | child_rss_MB: 22727.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:42:38,207] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 04:42:38,254] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:42:58,585] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 04:42:58,585] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:39,003] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 04:43:39,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:51,310] INFO : Step 35000/500000 rank0 | Train Loss: 1.7736 | Train Loss Unweighted: 1.1257 | Train Loss Components: Classification: 0.0007 | Classification_binary: 1.0265 | Regression_cos_phi: 0.0068 | Regression_energy: 0.0411 | Regression_eta: 0.0073 | Regression_pt: 0.0355 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0089 | Classification_binary: 1.0265 | Regression_cos_phi: 0.0598 | Regression_energy: 0.2468 | Regression_eta: 0.0460 | Regression_pt: 0.3166 | Regression_sin_phi: 0.0690 | LR: 8.16e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7517s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.1097 | max_tensorboard_logging: 0.2137 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2732.4 | uss_MB: 1490.0 | child_rss_MB: 22726.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:44:20,002] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 04:44:20,002] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:59,305] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 04:44:59,306] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:02,197] INFO : Step 36000/500000 rank0 | Train Loss: 0.8087 | Train Loss Unweighted: 0.3775 | Train Loss Components: Classification: 0.0045 | Classification_binary: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0204 | Regression_eta: 0.0062 | Regression_pt: 0.0257 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.3155 | Regression_cos_phi: 0.0271 | Regression_energy: 0.1226 | Regression_eta: 0.0390 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0196 | LR: 8.50e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7060s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0033 | max_backward: 0.0255 | max_optimizer_step: 0.0786 | max_tensorboard_logging: 0.2055 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2732.4 | uss_MB: 1490.0 | child_rss_MB: 22726.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:45:02,199] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 04:45:02,245] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:45:39,369] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 04:45:39,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:13,738] INFO : Step 37000/500000 rank0 | Train Loss: 2.5341 | Train Loss Unweighted: 1.6463 | Train Loss Components: Classification: 0.0032 | Classification_binary: 1.5142 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0505 | Regression_eta: 0.0081 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 1.5142 | Regression_cos_phi: 0.0322 | Regression_energy: 0.3035 | Regression_eta: 0.0514 | Regression_pt: 0.5007 | Regression_sin_phi: 0.0924 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7078s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0781 | max_tensorboard_logging: 0.2103 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2732.7 | uss_MB: 1490.3 | child_rss_MB: 22725.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:46:20,356] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 04:46:20,357] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:00,927] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 04:47:00,928] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:26,549] INFO : Step 38000/500000 rank0 | Train Loss: 1.2886 | Train Loss Unweighted: 0.6504 | Train Loss Components: Classification: 0.0085 | Classification_binary: 0.5603 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0283 | Regression_eta: 0.0084 | Regression_pt: 0.0351 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1056 | Classification_binary: 0.5603 | Regression_cos_phi: 0.0425 | Regression_energy: 0.1698 | Regression_eta: 0.0533 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0444 | LR: 9.21e-05 | DataLoad Time: 0.0091s | Model Forward Time: 0.7071s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0791 | max_tensorboard_logging: 0.2090 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2729.7 | uss_MB: 1487.3 | child_rss_MB: 22726.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:47:26,551] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 04:47:26,597] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:47:42,153] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 04:47:42,153] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:22,642] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 04:48:22,642] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:39,299] INFO : Step 39000/500000 rank0 | Train Loss: 1.7061 | Train Loss Unweighted: 1.0133 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.9081 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0388 | Regression_eta: 0.0110 | Regression_pt: 0.0406 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0079 | Classification_binary: 0.9081 | Regression_cos_phi: 0.0650 | Regression_energy: 0.2332 | Regression_eta: 0.0695 | Regression_pt: 0.3616 | Regression_sin_phi: 0.0608 | LR: 9.57e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7186s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0804 | max_tensorboard_logging: 0.2130 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2732.4 | uss_MB: 1490.0 | child_rss_MB: 22725.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:49:03,762] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 04:49:03,763] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:44,188] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 04:49:44,188] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:51,857] INFO : Step 40000/500000 rank0 | Train Loss: 0.9758 | Train Loss Unweighted: 0.3811 | Train Loss Components: Classification: 0.0052 | Classification_binary: 0.2926 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0382 | Regression_eta: 0.0047 | Regression_pt: 0.0349 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.2926 | Regression_cos_phi: 0.0254 | Regression_energy: 0.2293 | Regression_eta: 0.0297 | Regression_pt: 0.3113 | Regression_sin_phi: 0.0235 | LR: 9.94e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7025s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2123 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2732.8 | uss_MB: 1490.4 | child_rss_MB: 22724.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:49:51,859] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 04:49:51,906] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 04:49:51,907] INFO : Running validation on rank0 +rank0 [2026-11-14 04:49:51,907] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:49:51,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:49:51,987] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 04:49:52,061] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:49:52,129] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:49:52,177] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:49:52,222] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:49:52,267] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:49:52,313] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:49:52,359] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:49:52,431] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:49:52,500] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:49:52,567] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:49:52,613] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:49:52,658] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:49:52,703] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:49:52,749] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:49:52,795] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:49:52,867] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:49:52,871] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:49:52,873] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:49:52,874] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:49:52,876] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:49:52,877] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:49:52,879] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:49:52,880] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:49:52,882] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:49:52,883] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:49:52,885] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:49:52,886] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:49:52,887] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:49:52,888] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:49:52,889] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:49:52,891] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:49:53,823] INFO : VALIDATION | Step=40000/500000 | Train Loss=0.9758 | Valid Loss=10.4008 | Train Loss Unweighted=0.3811 | Valid Loss Unweighted=5.2852 | Stale=2 | ETA=559.4m +rank0 [2026-11-14 04:49:53,824] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:49:53,921] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:49:53,930] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:49:53,974] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:49:54,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:49:54,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:49:54,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:49:54,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:49:54,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:49:55,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:49:55,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:49:55,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:49:55,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:49:55,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:49:55,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:49:56,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:49:56,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:49:56,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:49:56,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:49:56,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:49:56,739] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:49:56,747] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:49:57,101] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:49:57,369] INFO : Plotted number of elements +rank0 [2026-11-14 04:49:58,251] INFO : Plotted jets +rank0 [2026-11-14 04:49:58,920] INFO : Plotted jet ratio +rank0 [2026-11-14 04:50:32,618] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 04:50:32,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:11,207] INFO : Step 41000/500000 rank0 | Train Loss: 1.3400 | Train Loss Unweighted: 0.6179 | Train Loss Components: Classification: 0.0015 | Classification_binary: 0.5080 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0495 | Regression_eta: 0.0031 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0183 | Classification_binary: 0.5080 | Regression_cos_phi: 0.0272 | Regression_energy: 0.2976 | Regression_eta: 0.0197 | Regression_pt: 0.4525 | Regression_sin_phi: 0.0166 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7273s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0298 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2179 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2756.6 | uss_MB: 1514.2 | child_rss_MB: 22724.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:51:13,019] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 04:51:13,019] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:53,158] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 04:51:53,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:23,493] INFO : Step 42000/500000 rank0 | Train Loss: 2.0359 | Train Loss Unweighted: 0.9379 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.7689 | Regression_cos_phi: 0.0036 | Regression_energy: 0.0770 | Regression_eta: 0.0066 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0096 | Classification_binary: 0.7689 | Regression_cos_phi: 0.0318 | Regression_energy: 0.4629 | Regression_eta: 0.0417 | Regression_pt: 0.7010 | Regression_sin_phi: 0.0200 | LR: 1.07e-04 | DataLoad Time: 0.0158s | Model Forward Time: 0.7118s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0268 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.2109 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2757.1 | uss_MB: 1514.8 | child_rss_MB: 22727.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:52:23,494] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 04:52:23,543] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 04:52:34,375] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 04:52:34,375] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:15,327] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 04:53:15,327] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:36,821] INFO : Step 43000/500000 rank0 | Train Loss: 1.4576 | Train Loss Unweighted: 0.5827 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.4478 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0629 | Regression_eta: 0.0040 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0061 | Classification_binary: 0.4478 | Regression_cos_phi: 0.0250 | Regression_energy: 0.3779 | Regression_eta: 0.0253 | Regression_pt: 0.5380 | Regression_sin_phi: 0.0375 | LR: 1.11e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7360s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0235 | max_loss: 0.0022 | max_backward: 0.0261 | max_optimizer_step: 0.1054 | max_tensorboard_logging: 0.2115 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2760.2 | uss_MB: 1517.8 | child_rss_MB: 22726.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:53:56,582] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 04:53:56,582] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:37,243] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 04:54:37,243] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:49,521] INFO : Step 44000/500000 rank0 | Train Loss: 0.8233 | Train Loss Unweighted: 0.3113 | Train Loss Components: Classification: 0.0011 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0066 | Regression_energy: 0.0302 | Regression_eta: 0.0056 | Regression_pt: 0.0307 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0137 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0584 | Regression_energy: 0.1814 | Regression_eta: 0.0355 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0266 | LR: 1.15e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7045s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0264 | max_loss: 0.0023 | max_backward: 0.0259 | max_optimizer_step: 0.0785 | max_tensorboard_logging: 0.2039 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2760.6 | uss_MB: 1518.2 | child_rss_MB: 22726.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:54:49,524] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 04:54:49,572] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 04:55:18,329] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 04:55:18,329] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:55:59,132] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 04:55:59,132] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:02,672] INFO : Step 45000/500000 rank0 | Train Loss: 1.2738 | Train Loss Unweighted: 0.2477 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.0883 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0778 | Regression_eta: 0.0038 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0017 | Classification_binary: 0.0883 | Regression_cos_phi: 0.0168 | Regression_energy: 0.4673 | Regression_eta: 0.0239 | Regression_pt: 0.6561 | Regression_sin_phi: 0.0198 | LR: 1.19e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7184s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0022 | max_backward: 0.0264 | max_optimizer_step: 0.0756 | max_tensorboard_logging: 0.2100 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2760.6 | uss_MB: 1518.2 | child_rss_MB: 22726.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:56:40,183] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 04:56:40,183] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:57:14,418] INFO : Step 46000/500000 rank0 | Train Loss: 0.8477 | Train Loss Unweighted: 0.1518 | Train Loss Components: Classification: 0.0093 | Classification_binary: 0.0505 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0407 | Regression_eta: 0.0076 | Regression_pt: 0.0376 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1159 | Classification_binary: 0.0505 | Regression_cos_phi: 0.0249 | Regression_energy: 0.2446 | Regression_eta: 0.0480 | Regression_pt: 0.3352 | Regression_sin_phi: 0.0286 | LR: 1.23e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7134s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0234 | max_loss: 0.0022 | max_backward: 0.0253 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2757.2 | uss_MB: 1514.8 | child_rss_MB: 22725.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:57:14,420] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 04:57:14,469] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 04:57:20,338] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 04:57:20,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:00,780] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 04:58:00,780] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:26,887] INFO : Step 47000/500000 rank0 | Train Loss: 1.2191 | Train Loss Unweighted: 0.4862 | Train Loss Components: Classification: 0.0012 | Classification_binary: 0.3746 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0491 | Regression_eta: 0.0042 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0146 | Classification_binary: 0.3746 | Regression_cos_phi: 0.0374 | Regression_energy: 0.2950 | Regression_eta: 0.0264 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0253 | LR: 1.27e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7269s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0791 | max_tensorboard_logging: 0.2106 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2760.2 | uss_MB: 1517.9 | child_rss_MB: 22727.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:58:41,818] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 04:58:41,818] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:22,231] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 04:59:22,231] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:39,439] INFO : Step 48000/500000 rank0 | Train Loss: 0.6138 | Train Loss Unweighted: 0.2734 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2211 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0205 | Regression_eta: 0.0052 | Regression_pt: 0.0217 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0021 | Classification_binary: 0.2211 | Regression_cos_phi: 0.0272 | Regression_energy: 0.1230 | Regression_eta: 0.0330 | Regression_pt: 0.1937 | Regression_sin_phi: 0.0138 | LR: 1.31e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7166s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0251 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2760.3 | uss_MB: 1517.9 | child_rss_MB: 22725.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:59:39,440] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 04:59:39,486] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:00:03,012] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:00:03,013] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:43,433] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:00:43,433] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:51,710] INFO : Step 49000/500000 rank0 | Train Loss: 2.7575 | Train Loss Unweighted: 0.7737 | Train Loss Components: Classification: 0.0013 | Classification_binary: 0.4690 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1420 | Regression_eta: 0.0068 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0158 | Classification_binary: 0.4690 | Regression_cos_phi: 0.0698 | Regression_energy: 0.8533 | Regression_eta: 0.0428 | Regression_pt: 1.2726 | Regression_sin_phi: 0.0344 | LR: 1.36e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7138s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0787 | max_tensorboard_logging: 0.2155 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2760.5 | uss_MB: 1518.2 | child_rss_MB: 22724.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:01:24,574] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:01:24,575] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:04,592] INFO : Step 50000/500000 rank0 | Train Loss: 1.5569 | Train Loss Unweighted: 0.8630 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.7554 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0494 | Regression_eta: 0.0056 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0035 | Classification_binary: 0.7554 | Regression_cos_phi: 0.0177 | Regression_energy: 0.2966 | Regression_eta: 0.0352 | Regression_pt: 0.4174 | Regression_sin_phi: 0.0310 | LR: 1.40e-04 | DataLoad Time: 0.0052s | Model Forward Time: 0.7122s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0262 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2756.8 | uss_MB: 1514.4 | child_rss_MB: 22726.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:02:04,594] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:02:04,641] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:02:04,641] INFO : Running validation on rank0 +rank0 [2026-11-14 05:02:04,642] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:02:04,642] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:02:04,723] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:02:04,796] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:02:04,863] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:02:04,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:02:04,958] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:02:05,004] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:02:05,050] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:02:05,095] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:02:05,169] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:02:05,238] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:02:05,306] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:02:05,352] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:02:05,397] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:02:05,442] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:02:05,489] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:02:05,533] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:02:05,598] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:02:05,602] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:02:05,603] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:02:05,605] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:02:05,606] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:02:05,608] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:02:05,609] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:02:05,611] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:02:05,612] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:02:05,614] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:02:05,615] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:02:05,616] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:02:05,618] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:02:05,618] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:02:05,620] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:02:05,621] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:02:06,516] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.5569 | Valid Loss=11.4547 | Train Loss Unweighted=0.8630 | Valid Loss Unweighted=6.3916 | Stale=3 | ETA=547.7m +rank0 [2026-11-14 05:02:06,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:02:06,612] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:02:06,620] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:02:06,663] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:02:07,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:02:07,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:02:07,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:02:07,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:02:07,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:02:07,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:02:07,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:02:08,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:02:08,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:02:08,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:02:08,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:02:08,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:02:08,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:02:09,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:02:09,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:02:09,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:02:09,411] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 05:02:09,417] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:02:09,780] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:02:10,047] INFO : Plotted number of elements +rank0 [2026-11-14 05:02:10,921] INFO : Plotted jets +rank0 [2026-11-14 05:02:11,582] INFO : Plotted jet ratio +rank0 [2026-11-14 05:02:12,037] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:02:12,037] INFO : Training completed. Total time on device 0: 60.951min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..5f0cc4ba48a4e1b0dfe74e05045cce0cbafc366b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/train.log.0 @@ -0,0 +1,997 @@ +rank0 [2026-11-14 04:01:12,418] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 04:01:12,418] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 04:01:12,419] INFO : Instantiating model +rank0 [2026-11-14 04:01:12,437] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 04:01:12,437] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 04:01:12,439] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 04:01:12,439] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 04:01:12,439] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 04:01:12,439] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 04:01:12,439] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,441] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 04:01:12,441] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,443] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 04:01:12,443] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,445] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 04:01:12,446] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,447] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 04:01:12,447] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,450] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 04:01:12,450] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,452] INFO : Backbone initialization took 0.01s +rank0 [2026-11-14 04:01:12,452] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 04:01:12,452] INFO : Initializing output DNNs +rank0 [2026-11-14 04:01:12,453] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,456] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,462] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 04:01:12,462] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 04:01:12,462] INFO : jagged_attention enabled=True +rank0 [2026-11-14 04:01:12,462] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 04:01:12,462] INFO : task_queries enabled=True +rank0 [2026-11-14 04:01:12,462] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 04:01:12,462] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 04:01:12,462] INFO : classification_query parameters: 256 +rank0 [2026-11-14 04:01:12,462] INFO : regression_query parameters: 256 +rank0 [2026-11-14 04:01:12,463] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 04:01:12,463] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 04:01:12,463] INFO : output_mode=elementwise +rank0 [2026-11-14 04:01:12,463] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 04:01:12,463] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 04:01:12,463] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 04:01:12,463] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 04:01:12,463] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 04:01:12,463] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 04:01:12,463] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 04:01:12,463] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 04:01:12,464] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 04:01:12,464] INFO : MLPF __init__ done +rank0 [2026-11-14 04:01:12,464] INFO : Instantiated model +rank0 [2026-11-14 04:01:12,464] INFO : Moving model to device rank=0 +rank0 [2026-11-14 04:01:12,470] INFO : Moved model to device rank=0 +rank0 [2026-11-14 04:01:12,471] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 04:01:12,537] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 04:01:12,537] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 04:01:12,538] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 04:01:12,538] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 04:01:12,538] INFO : Total parameters: 3910930 +rank0 [2026-11-14 04:01:12,539] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 04:01:12,541] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903 +rank0 [2026-11-14 04:01:12,541] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903 +rank0 [2026-11-14 04:01:12,541] INFO : Getting dataloaders +rank0 [2026-11-14 04:01:12,541] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:01:14,177] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,217] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,256] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,298] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,336] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,381] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,455] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,500] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,590] INFO : train_dataset: cld_edm_z_tautau_pf, 898 +rank0 [2026-11-14 04:01:14,659] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,659] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 04:01:14,659] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 04:01:14,659] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 04:01:14,659] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:01:14,669] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,679] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,689] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,699] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,709] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,718] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,728] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,738] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,747] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,757] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,757] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 04:01:14,757] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 04:01:14,792] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:01:14,994] INFO : Got dataloaders +rank0 [2026-11-14 04:01:14,994] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 04:01:14,994] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 04:01:14,995] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 04:01:15,003] INFO : Creating train_iterator +rank0 [2026-11-14 04:01:15,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:15,038] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:01:15,225] INFO : Created train_iterator +rank0 [2026-11-14 04:01:23,348] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.436795501708985, 'Classification': 0.598533673286438, 'Regression_pt': 0.8346810086071491, 'Regression_eta': 1.176606583595276, 'Regression_sin_phi': 0.8390423528850078, 'Regression_cos_phi': 0.8456102706491947, 'Regression_energy': 1.2376715517044068}, weights={'Classification_binary': 1.0, 'Classification': 12.425024032592773, 'Regression_pt': 8.909745216369629, 'Regression_eta': 6.320545673370361, 'Regression_sin_phi': 8.863432884216309, 'Regression_cos_phi': 8.79458999633789, 'Regression_energy': 6.0086989402771} +rank0 [2026-11-14 04:01:56,726] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 04:01:56,726] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:02:28,485] INFO : Step 1000/500000 rank0 | Train Loss: 10.8482 | Train Loss Unweighted: 5.2339 | Train Loss Components: Classification: 0.0953 | Classification_binary: 4.4474 | Regression_cos_phi: 0.0295 | Regression_energy: 0.2843 | Regression_eta: 0.0435 | Regression_pt: 0.3069 | Regression_sin_phi: 0.0271 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 1.1841 | Classification_binary: 4.4474 | Regression_cos_phi: 0.2596 | Regression_energy: 1.7080 | Regression_eta: 0.2752 | Regression_pt: 2.7340 | Regression_sin_phi: 0.2399 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7171s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0250 | max_optimizer_step: 0.0780 | max_tensorboard_logging: 0.2286 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2589.8 | uss_MB: 1340.5 | child_rss_MB: 22726.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:02:37,106] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:02:37,106] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:16,563] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:03:16,563] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:39,547] INFO : Step 2000/500000 rank0 | Train Loss: 8.6441 | Train Loss Unweighted: 3.4456 | Train Loss Components: Classification: 0.0401 | Classification_binary: 2.6864 | Regression_cos_phi: 0.0118 | Regression_energy: 0.3166 | Regression_eta: 0.0105 | Regression_pt: 0.3674 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.4982 | Classification_binary: 2.6864 | Regression_cos_phi: 0.1039 | Regression_energy: 1.9025 | Regression_eta: 0.0661 | Regression_pt: 3.2738 | Regression_sin_phi: 0.1132 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.6989s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0247 | max_optimizer_step: 0.0762 | max_tensorboard_logging: 0.2160 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2590.5 | uss_MB: 1340.4 | child_rss_MB: 22722.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:03:39,549] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:03:56,665] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:03:56,665] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:36,999] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:04:36,999] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:50,856] INFO : Step 3000/500000 rank0 | Train Loss: 13.6524 | Train Loss Unweighted: 5.7479 | Train Loss Components: Classification: 0.0648 | Classification_binary: 4.5797 | Regression_cos_phi: 0.0296 | Regression_energy: 0.5128 | Regression_eta: 0.0275 | Regression_pt: 0.5178 | Regression_sin_phi: 0.0158 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.8046 | Classification_binary: 4.5797 | Regression_cos_phi: 0.2601 | Regression_energy: 3.0812 | Regression_eta: 0.1738 | Regression_pt: 4.6132 | Regression_sin_phi: 0.1399 | LR: 2.05e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.6969s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0762 | max_tensorboard_logging: 0.2176 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2622.4 | uss_MB: 1372.6 | child_rss_MB: 22722.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:05:17,262] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:05:17,262] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:05:56,746] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:05:56,747] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:02,045] INFO : Step 4000/500000 rank0 | Train Loss: 4.7707 | Train Loss Unweighted: 3.1381 | Train Loss Components: Classification: 0.0355 | Classification_binary: 2.9134 | Regression_cos_phi: 0.0075 | Regression_energy: 0.0836 | Regression_eta: 0.0100 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.4412 | Classification_binary: 2.9134 | Regression_cos_phi: 0.0659 | Regression_energy: 0.5024 | Regression_eta: 0.0629 | Regression_pt: 0.7231 | Regression_sin_phi: 0.0619 | LR: 2.08e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6936s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0231 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0764 | max_tensorboard_logging: 0.2148 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2622.4 | uss_MB: 1372.6 | child_rss_MB: 22722.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:06:02,049] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:06:37,344] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:06:37,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:13,913] INFO : Step 5000/500000 rank0 | Train Loss: 19.4153 | Train Loss Unweighted: 5.5590 | Train Loss Components: Classification: 0.0360 | Classification_binary: 3.4586 | Regression_cos_phi: 0.0315 | Regression_energy: 0.9579 | Regression_eta: 0.0370 | Regression_pt: 0.9176 | Regression_sin_phi: 0.1204 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.4479 | Classification_binary: 3.4586 | Regression_cos_phi: 0.2769 | Regression_energy: 5.7557 | Regression_eta: 0.2338 | Regression_pt: 8.1755 | Regression_sin_phi: 1.0670 | LR: 2.13e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7008s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0230 | max_loss: 0.0022 | max_backward: 0.0255 | max_optimizer_step: 0.0760 | max_tensorboard_logging: 0.2215 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2592.6 | uss_MB: 1342.8 | child_rss_MB: 22722.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:07:18,002] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:07:18,002] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:57,754] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:07:57,754] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:25,373] INFO : Step 6000/500000 rank0 | Train Loss: 9.5847 | Train Loss Unweighted: 4.5101 | Train Loss Components: Classification: 0.0145 | Classification_binary: 3.7352 | Regression_cos_phi: 0.0278 | Regression_energy: 0.3257 | Regression_eta: 0.0603 | Regression_pt: 0.3043 | Regression_sin_phi: 0.0424 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1803 | Classification_binary: 3.7352 | Regression_cos_phi: 0.2441 | Regression_energy: 1.9570 | Regression_eta: 0.3809 | Regression_pt: 2.7112 | Regression_sin_phi: 0.3759 | LR: 2.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.6980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0231 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0778 | max_tensorboard_logging: 0.2128 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2592.7 | uss_MB: 1342.9 | child_rss_MB: 22723.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:08:25,375] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:08:38,325] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:08:38,325] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:18,273] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:09:18,273] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:36,719] INFO : Step 7000/500000 rank0 | Train Loss: 4.2768 | Train Loss Unweighted: 2.2646 | Train Loss Components: Classification: 0.0196 | Classification_binary: 1.9713 | Regression_cos_phi: 0.0037 | Regression_energy: 0.1205 | Regression_eta: 0.0100 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.2433 | Classification_binary: 1.9713 | Regression_cos_phi: 0.0324 | Regression_energy: 0.7238 | Regression_eta: 0.0632 | Regression_pt: 1.1511 | Regression_sin_phi: 0.0917 | LR: 2.26e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7031s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0249 | max_optimizer_step: 0.0759 | max_tensorboard_logging: 0.2178 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2593.3 | uss_MB: 1343.5 | child_rss_MB: 22722.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:09:58,242] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:09:58,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:37,978] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:10:37,978] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:47,944] INFO : Step 8000/500000 rank0 | Train Loss: 8.6984 | Train Loss Unweighted: 3.1007 | Train Loss Components: Classification: 0.0496 | Classification_binary: 2.2744 | Regression_cos_phi: 0.0114 | Regression_energy: 0.3773 | Regression_eta: 0.0063 | Regression_pt: 0.3770 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.6157 | Classification_binary: 2.2744 | Regression_cos_phi: 0.1006 | Regression_energy: 2.2669 | Regression_eta: 0.0396 | Regression_pt: 3.3590 | Regression_sin_phi: 0.0422 | LR: 2.34e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6902s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0256 | max_optimizer_step: 0.0781 | max_tensorboard_logging: 0.2098 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2593.3 | uss_MB: 1343.5 | child_rss_MB: 22721.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:10:47,946] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:10:48,006] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:11:18,552] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:11:18,552] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:58,615] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:11:58,615] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:59,834] INFO : Step 9000/500000 rank0 | Train Loss: 5.1573 | Train Loss Unweighted: 2.1450 | Train Loss Components: Classification: 0.0071 | Classification_binary: 1.6877 | Regression_cos_phi: 0.0055 | Regression_energy: 0.2048 | Regression_eta: 0.0133 | Regression_pt: 0.2224 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 1.6877 | Regression_cos_phi: 0.0481 | Regression_energy: 1.2305 | Regression_eta: 0.0838 | Regression_pt: 1.9814 | Regression_sin_phi: 0.0371 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.6944s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0231 | max_loss: 0.0022 | max_backward: 0.0257 | max_optimizer_step: 0.0765 | max_tensorboard_logging: 0.2138 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2591.3 | uss_MB: 1341.6 | child_rss_MB: 22724.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:12:39,251] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:12:39,251] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:11,869] INFO : Step 10000/500000 rank0 | Train Loss: 4.7017 | Train Loss Unweighted: 2.6628 | Train Loss Components: Classification: 0.0261 | Classification_binary: 2.3757 | Regression_cos_phi: 0.0193 | Regression_energy: 0.1010 | Regression_eta: 0.0106 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0304 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.3248 | Classification_binary: 2.3757 | Regression_cos_phi: 0.1698 | Regression_energy: 0.6070 | Regression_eta: 0.0669 | Regression_pt: 0.8880 | Regression_sin_phi: 0.2694 | LR: 2.52e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6904s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0230 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.2115 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2591.9 | uss_MB: 1342.0 | child_rss_MB: 22724.7 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:13:11,871] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:13:11,920] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:13:11,921] INFO : Running validation on rank0 +rank0 [2026-11-14 04:13:11,921] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:13:11,921] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:13:12,052] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:13:12,183] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:13:12,232] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:13:12,270] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:13:12,309] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:13:12,342] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:13:12,381] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:13:12,419] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:13:12,453] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:13:12,499] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:13:12,545] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:13:12,583] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:13:12,622] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:13:12,655] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:13:12,693] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:13:12,732] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:13:12,768] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:13:12,770] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:13:12,771] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:13:12,773] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:13:12,774] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:13:12,776] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:13:12,777] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:13:12,779] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:13:12,780] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:13:12,782] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:13:12,783] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:13:12,785] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:13:12,786] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:13:12,786] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:13:12,788] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:13:12,789] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:13:13,742] INFO : VALIDATION | Step=10000/500000 | Train Loss=4.7017 | Valid Loss=7.5622 | Train Loss Unweighted=2.6628 | Valid Loss Unweighted=3.1410 | Stale=0 | ETA=587.0m +rank0 [2026-11-14 04:13:13,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:13:13,840] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:13:13,849] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:13,902] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:13:14,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:13:14,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:13:14,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:13:15,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:13:15,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:13:15,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:13:15,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:13:15,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:13:15,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:13:15,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:13:16,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:13:16,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:13:16,417] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:13:16,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:13:16,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:13:16,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:13:16,966] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 04:13:16,973] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:17,325] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:18,030] INFO : Plotted number of elements +rank0 [2026-11-14 04:13:18,941] INFO : Plotted jets +rank0 [2026-11-14 04:13:19,605] INFO : Plotted jet ratio +rank0 [2026-11-14 04:13:28,360] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:13:28,360] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:08,684] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:14:08,684] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:32,491] INFO : Step 11000/500000 rank0 | Train Loss: 7.1059 | Train Loss Unweighted: 2.3937 | Train Loss Components: Classification: 0.0049 | Classification_binary: 1.6778 | Regression_cos_phi: 0.0192 | Regression_energy: 0.3223 | Regression_eta: 0.0114 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0249 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 1.6778 | Regression_cos_phi: 0.1690 | Regression_energy: 1.9366 | Regression_eta: 0.0724 | Regression_pt: 2.9685 | Regression_sin_phi: 0.2210 | LR: 2.63e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7242s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0262 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2155 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.1 | uss_MB: 1483.6 | child_rss_MB: 22724.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:14:49,751] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:14:49,751] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:30,083] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:15:30,084] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:44,917] INFO : Step 12000/500000 rank0 | Train Loss: 3.0135 | Train Loss Unweighted: 1.8501 | Train Loss Components: Classification: 0.0091 | Classification_binary: 1.6765 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0728 | Regression_eta: 0.0116 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1132 | Classification_binary: 1.6765 | Regression_cos_phi: 0.0476 | Regression_energy: 0.4377 | Regression_eta: 0.0732 | Regression_pt: 0.6259 | Regression_sin_phi: 0.0393 | LR: 2.75e-05 | DataLoad Time: 0.0031s | Model Forward Time: 0.7123s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0263 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2119 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2726.2 | uss_MB: 1483.7 | child_rss_MB: 22723.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:15:44,919] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:15:44,967] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:16:11,028] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:16:11,028] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:16:51,730] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:16:51,730] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:16:57,717] INFO : Step 13000/500000 rank0 | Train Loss: 8.9040 | Train Loss Unweighted: 3.5480 | Train Loss Components: Classification: 0.0152 | Classification_binary: 2.7349 | Regression_cos_phi: 0.0114 | Regression_energy: 0.3668 | Regression_eta: 0.0245 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1891 | Classification_binary: 2.7349 | Regression_cos_phi: 0.1001 | Regression_energy: 2.2037 | Regression_eta: 0.1548 | Regression_pt: 3.3933 | Regression_sin_phi: 0.1281 | LR: 2.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7107s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0776 | max_tensorboard_logging: 0.2142 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2671.4 | uss_MB: 1428.9 | child_rss_MB: 22722.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:17:32,786] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:17:32,786] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:10,207] INFO : Step 14000/500000 rank0 | Train Loss: 3.4496 | Train Loss Unweighted: 1.9632 | Train Loss Components: Classification: 0.0042 | Classification_binary: 1.7365 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1025 | Regression_eta: 0.0090 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 1.7365 | Regression_cos_phi: 0.0604 | Regression_energy: 0.6158 | Regression_eta: 0.0568 | Regression_pt: 0.8793 | Regression_sin_phi: 0.0487 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7052s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2671.9 | uss_MB: 1429.5 | child_rss_MB: 22722.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:18:10,208] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:18:10,256] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:18:13,874] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:18:13,874] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:54,357] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:18:54,357] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:23,207] INFO : Step 15000/500000 rank0 | Train Loss: 6.1661 | Train Loss Unweighted: 2.7591 | Train Loss Components: Classification: 0.0108 | Classification_binary: 2.2285 | Regression_cos_phi: 0.0045 | Regression_energy: 0.2574 | Regression_eta: 0.0306 | Regression_pt: 0.2118 | Regression_sin_phi: 0.0154 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1339 | Classification_binary: 2.2285 | Regression_cos_phi: 0.0399 | Regression_energy: 1.5465 | Regression_eta: 0.1936 | Regression_pt: 1.8871 | Regression_sin_phi: 0.1365 | LR: 3.17e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7072s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2128 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2675.3 | uss_MB: 1432.9 | child_rss_MB: 22723.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:19:35,681] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:19:35,682] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:16,141] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:20:16,141] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:35,746] INFO : Step 16000/500000 rank0 | Train Loss: 9.1400 | Train Loss Unweighted: 2.9519 | Train Loss Components: Classification: 0.0127 | Classification_binary: 1.9928 | Regression_cos_phi: 0.0133 | Regression_energy: 0.4415 | Regression_eta: 0.0615 | Regression_pt: 0.3999 | Regression_sin_phi: 0.0301 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1584 | Classification_binary: 1.9928 | Regression_cos_phi: 0.1170 | Regression_energy: 2.6529 | Regression_eta: 0.3889 | Regression_pt: 3.5630 | Regression_sin_phi: 0.2670 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7074s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.2146 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2675.3 | uss_MB: 1432.9 | child_rss_MB: 22723.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:20:35,748] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:20:35,796] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:20:57,212] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:20:57,212] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:37,641] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:21:37,641] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:48,178] INFO : Step 17000/500000 rank0 | Train Loss: 3.0402 | Train Loss Unweighted: 1.8028 | Train Loss Components: Classification: 0.0132 | Classification_binary: 1.6237 | Regression_cos_phi: 0.0184 | Regression_energy: 0.0646 | Regression_eta: 0.0137 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1645 | Classification_binary: 1.6237 | Regression_cos_phi: 0.1616 | Regression_energy: 0.3879 | Regression_eta: 0.0867 | Regression_pt: 0.5304 | Regression_sin_phi: 0.0853 | LR: 3.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7020s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0782 | max_tensorboard_logging: 0.2080 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2671.3 | uss_MB: 1428.9 | child_rss_MB: 22724.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:22:18,870] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:22:18,871] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:59,165] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:22:59,165] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:00,904] INFO : Step 18000/500000 rank0 | Train Loss: 4.1121 | Train Loss Unweighted: 2.4614 | Train Loss Components: Classification: 0.0067 | Classification_binary: 2.2116 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1136 | Regression_eta: 0.0070 | Regression_pt: 0.1103 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0833 | Classification_binary: 2.2116 | Regression_cos_phi: 0.0615 | Regression_energy: 0.6829 | Regression_eta: 0.0440 | Regression_pt: 0.9827 | Regression_sin_phi: 0.0462 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.6940s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0233 | max_loss: 0.0022 | max_backward: 0.0250 | max_optimizer_step: 0.0783 | max_tensorboard_logging: 0.1987 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2672.1 | uss_MB: 1429.6 | child_rss_MB: 22723.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:23:00,906] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:23:00,953] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:23:39,328] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:23:39,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:12,310] INFO : Step 19000/500000 rank0 | Train Loss: 4.0787 | Train Loss Unweighted: 2.2921 | Train Loss Components: Classification: 0.0068 | Classification_binary: 2.0221 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1199 | Regression_eta: 0.0093 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0846 | Classification_binary: 2.0221 | Regression_cos_phi: 0.0296 | Regression_energy: 0.7207 | Regression_eta: 0.0586 | Regression_pt: 1.0965 | Regression_sin_phi: 0.0665 | LR: 3.88e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7337s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0291 | max_loss: 0.0022 | max_backward: 0.0254 | max_optimizer_step: 0.0785 | max_tensorboard_logging: 0.2023 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2676.1 | uss_MB: 1433.7 | child_rss_MB: 22724.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:19,908] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:24:19,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:59,255] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:24:59,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:23,999] INFO : Step 20000/500000 rank0 | Train Loss: 4.1707 | Train Loss Unweighted: 2.7973 | Train Loss Components: Classification: 0.0080 | Classification_binary: 2.5912 | Regression_cos_phi: 0.0109 | Regression_energy: 0.0845 | Regression_eta: 0.0146 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0990 | Classification_binary: 2.5912 | Regression_cos_phi: 0.0955 | Regression_energy: 0.5079 | Regression_eta: 0.0925 | Regression_pt: 0.6974 | Regression_sin_phi: 0.0871 | LR: 4.07e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7150s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0795 | max_tensorboard_logging: 0.2116 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2677.0 | uss_MB: 1434.6 | child_rss_MB: 22724.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:25:24,001] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:25:24,048] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:25:24,049] INFO : Running validation on rank0 +rank0 [2026-11-14 04:25:24,049] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:25:24,050] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:25:24,128] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:25:24,201] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:25:24,270] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:25:24,317] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:25:24,363] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:25:24,409] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:25:24,455] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:25:24,500] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:25:24,569] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:25:24,637] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:25:24,703] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:25:24,749] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:25:24,794] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:25:24,838] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:25:24,883] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:25:24,927] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:25:24,993] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:25:24,997] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:25:24,999] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:25:25,000] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:25:25,002] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:25:25,003] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:25:25,005] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:25:25,006] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:25:25,008] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:25:25,009] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:25:25,009] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:25:25,011] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:25:25,012] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:25:25,013] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:25:25,014] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:25:25,016] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:25:25,984] INFO : VALIDATION | Step=20000/500000 | Train Loss=4.1707 | Valid Loss=7.0211 | Train Loss Unweighted=2.7973 | Valid Loss Unweighted=3.0092 | Stale=0 | ETA=580.4m +rank0 [2026-11-14 04:25:25,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:25:26,076] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:25:26,085] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:26,128] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:25:26,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:25:26,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:25:26,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:25:26,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:25:27,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:25:27,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:25:27,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:25:27,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:25:27,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:25:27,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:25:28,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:25:28,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:25:28,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:25:28,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:25:28,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:25:28,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:25:28,873] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:25:28,879] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:29,232] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:29,504] INFO : Plotted number of elements +rank0 [2026-11-14 04:25:30,382] INFO : Plotted jets +rank0 [2026-11-14 04:25:31,059] INFO : Plotted jet ratio +rank0 [2026-11-14 04:25:48,266] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:25:48,266] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:28,637] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:26:28,637] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:44,150] INFO : Step 21000/500000 rank0 | Train Loss: 7.6543 | Train Loss Unweighted: 3.7666 | Train Loss Components: Classification: 0.0131 | Classification_binary: 3.1977 | Regression_cos_phi: 0.1132 | Regression_energy: 0.2088 | Regression_eta: 0.0151 | Regression_pt: 0.2125 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1625 | Classification_binary: 3.1977 | Regression_cos_phi: 0.9955 | Regression_energy: 1.2544 | Regression_eta: 0.0957 | Regression_pt: 1.8936 | Regression_sin_phi: 0.0549 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0234 | max_loss: 0.0022 | max_backward: 0.0260 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2297 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.3 | uss_MB: 1461.9 | child_rss_MB: 22725.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:27:09,819] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:27:09,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:50,277] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:27:50,278] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:56,856] INFO : Step 22000/500000 rank0 | Train Loss: 2.8379 | Train Loss Unweighted: 1.6083 | Train Loss Components: Classification: 0.0094 | Classification_binary: 1.4284 | Regression_cos_phi: 0.0055 | Regression_energy: 0.0743 | Regression_eta: 0.0039 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1171 | Classification_binary: 1.4284 | Regression_cos_phi: 0.0487 | Regression_energy: 0.4465 | Regression_eta: 0.0246 | Regression_pt: 0.7404 | Regression_sin_phi: 0.0323 | LR: 4.50e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7176s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.0806 | max_tensorboard_logging: 0.2118 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2705.3 | uss_MB: 1462.9 | child_rss_MB: 22723.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:27:56,858] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:27:56,905] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:31,802] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:28:31,802] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:09,962] INFO : Step 23000/500000 rank0 | Train Loss: 4.9195 | Train Loss Unweighted: 2.3961 | Train Loss Components: Classification: 0.0083 | Classification_binary: 2.0136 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1783 | Regression_eta: 0.0052 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1033 | Classification_binary: 2.0136 | Regression_cos_phi: 0.0400 | Regression_energy: 1.0716 | Regression_eta: 0.0331 | Regression_pt: 1.6130 | Regression_sin_phi: 0.0450 | LR: 4.73e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7181s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0784 | max_tensorboard_logging: 0.2173 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2708.7 | uss_MB: 1466.3 | child_rss_MB: 22723.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:29:12,997] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:29:12,998] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:53,645] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:29:53,645] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:22,880] INFO : Step 24000/500000 rank0 | Train Loss: 3.5262 | Train Loss Unweighted: 2.3697 | Train Loss Components: Classification: 0.0108 | Classification_binary: 2.2038 | Regression_cos_phi: 0.0091 | Regression_energy: 0.0591 | Regression_eta: 0.0081 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1347 | Classification_binary: 2.2038 | Regression_cos_phi: 0.0803 | Regression_energy: 0.3553 | Regression_eta: 0.0510 | Regression_pt: 0.6453 | Regression_sin_phi: 0.0557 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7126s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0233 | max_loss: 0.0022 | max_backward: 0.0260 | max_optimizer_step: 0.0786 | max_tensorboard_logging: 0.2142 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2708.7 | uss_MB: 1466.3 | child_rss_MB: 22723.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:30:22,881] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:30:22,929] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:30:34,919] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:30:34,919] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:15,469] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:31:15,469] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:36,128] INFO : Step 25000/500000 rank0 | Train Loss: 2.9109 | Train Loss Unweighted: 1.8483 | Train Loss Components: Classification: 0.0086 | Classification_binary: 1.6940 | Regression_cos_phi: 0.0139 | Regression_energy: 0.0599 | Regression_eta: 0.0047 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1067 | Classification_binary: 1.6940 | Regression_cos_phi: 0.1225 | Regression_energy: 0.3599 | Regression_eta: 0.0295 | Regression_pt: 0.5461 | Regression_sin_phi: 0.0523 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7057s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0236 | max_loss: 0.0022 | max_backward: 0.0262 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.2088 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.3 | uss_MB: 1461.9 | child_rss_MB: 22722.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:31:57,038] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:31:57,038] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:37,547] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:32:37,547] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:48,901] INFO : Step 26000/500000 rank0 | Train Loss: 1.9940 | Train Loss Unweighted: 1.3781 | Train Loss Components: Classification: 0.0013 | Classification_binary: 1.2876 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0290 | Regression_eta: 0.0078 | Regression_pt: 0.0387 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0164 | Classification_binary: 1.2876 | Regression_cos_phi: 0.0246 | Regression_energy: 0.1745 | Regression_eta: 0.0494 | Regression_pt: 0.3452 | Regression_sin_phi: 0.0963 | LR: 5.47e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2206 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.8 | uss_MB: 1462.4 | child_rss_MB: 22724.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:32:48,903] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:32:48,952] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:33:18,782] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:33:18,782] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:59,313] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:33:59,313] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:01,721] INFO : Step 27000/500000 rank0 | Train Loss: 2.9179 | Train Loss Unweighted: 1.2410 | Train Loss Components: Classification: 0.0051 | Classification_binary: 0.9867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.1180 | Regression_eta: 0.0038 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.9867 | Regression_cos_phi: 0.0226 | Regression_energy: 0.7092 | Regression_eta: 0.0237 | Regression_pt: 1.0885 | Regression_sin_phi: 0.0234 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7036s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0241 | max_loss: 0.0022 | max_backward: 0.0264 | max_optimizer_step: 0.0774 | max_tensorboard_logging: 0.2095 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2708.8 | uss_MB: 1466.4 | child_rss_MB: 22724.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:34:40,798] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:34:40,798] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:14,695] INFO : Step 28000/500000 rank0 | Train Loss: 4.9368 | Train Loss Unweighted: 2.0786 | Train Loss Components: Classification: 0.0261 | Classification_binary: 1.6569 | Regression_cos_phi: 0.0027 | Regression_energy: 0.1906 | Regression_eta: 0.0062 | Regression_pt: 0.1921 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.3239 | Classification_binary: 1.6569 | Regression_cos_phi: 0.0238 | Regression_energy: 1.1451 | Regression_eta: 0.0393 | Regression_pt: 1.7118 | Regression_sin_phi: 0.0359 | LR: 6.01e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7113s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0278 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2119 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2709.0 | uss_MB: 1466.6 | child_rss_MB: 22726.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:14,697] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:35:14,743] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:35:21,953] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:35:21,953] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:02,465] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:36:02,465] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:27,494] INFO : Step 29000/500000 rank0 | Train Loss: 3.8558 | Train Loss Unweighted: 1.7969 | Train Loss Components: Classification: 0.0061 | Classification_binary: 1.4840 | Regression_cos_phi: 0.0041 | Regression_energy: 0.1465 | Regression_eta: 0.0047 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 1.4840 | Regression_cos_phi: 0.0358 | Regression_energy: 0.8801 | Regression_eta: 0.0299 | Regression_pt: 1.3049 | Regression_sin_phi: 0.0452 | LR: 6.29e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7062s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0022 | max_backward: 0.0260 | max_optimizer_step: 0.0784 | max_tensorboard_logging: 0.2078 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2709.8 | uss_MB: 1467.4 | child_rss_MB: 22725.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:36:43,488] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:36:43,488] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:23,683] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:37:23,683] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:40,045] INFO : Step 30000/500000 rank0 | Train Loss: 2.7806 | Train Loss Unweighted: 1.9047 | Train Loss Components: Classification: 0.0042 | Classification_binary: 1.7727 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0513 | Regression_eta: 0.0128 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 1.7727 | Regression_cos_phi: 0.0466 | Regression_energy: 0.3081 | Regression_eta: 0.0811 | Regression_pt: 0.4403 | Regression_sin_phi: 0.0794 | LR: 6.58e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7115s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0251 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2112 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2704.9 | uss_MB: 1462.4 | child_rss_MB: 22725.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:37:40,047] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:37:40,096] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:37:40,096] INFO : Running validation on rank0 +rank0 [2026-11-14 04:37:40,096] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:37:40,096] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:37:40,203] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:37:40,277] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:37:40,346] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:37:40,394] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:37:40,440] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:37:40,486] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:37:40,532] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:37:40,579] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:37:40,653] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:37:40,721] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:37:40,789] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:37:40,835] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:37:40,881] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:37:40,926] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:37:40,971] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:37:41,017] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:37:41,089] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:37:41,093] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:37:41,095] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:37:41,097] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:37:41,098] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:37:41,100] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:37:41,101] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:37:41,103] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:37:41,104] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:37:41,106] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:37:41,107] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:37:41,108] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:37:41,110] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:37:41,110] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:37:41,112] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:37:41,113] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:37:42,028] INFO : VALIDATION | Step=30000/500000 | Train Loss=2.7806 | Valid Loss=7.8145 | Train Loss Unweighted=1.9047 | Valid Loss Unweighted=3.3596 | Stale=1 | ETA=571.1m +rank0 [2026-11-14 04:37:42,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:37:42,137] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:37:42,146] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:37:42,207] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:37:42,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:37:42,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:37:43,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:37:43,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:37:43,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:37:43,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:37:43,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:37:43,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:37:43,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:37:44,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:37:44,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:37:44,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:37:44,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:37:44,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:37:44,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:37:44,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:37:45,087] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:37:45,094] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:37:45,452] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:37:45,710] INFO : Plotted number of elements +rank0 [2026-11-14 04:37:46,576] INFO : Plotted jets +rank0 [2026-11-14 04:37:47,263] INFO : Plotted jet ratio +rank0 [2026-11-14 04:38:12,405] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:38:12,405] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:38:53,024] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:38:53,024] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:00,091] INFO : Step 31000/500000 rank0 | Train Loss: 1.4948 | Train Loss Unweighted: 0.8320 | Train Loss Components: Classification: 0.0072 | Classification_binary: 0.7342 | Regression_cos_phi: 0.0056 | Regression_energy: 0.0381 | Regression_eta: 0.0097 | Regression_pt: 0.0338 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0898 | Classification_binary: 0.7342 | Regression_cos_phi: 0.0493 | Regression_energy: 0.2287 | Regression_eta: 0.0613 | Regression_pt: 0.3014 | Regression_sin_phi: 0.0301 | LR: 6.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7328s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0793 | max_tensorboard_logging: 0.2171 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2822.7 | uss_MB: 1580.3 | child_rss_MB: 22724.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:39:34,137] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:39:34,137] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:13,014] INFO : Step 32000/500000 rank0 | Train Loss: 2.0666 | Train Loss Unweighted: 0.9571 | Train Loss Components: Classification: 0.0099 | Classification_binary: 0.7926 | Regression_cos_phi: 0.0077 | Regression_energy: 0.0687 | Regression_eta: 0.0101 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1230 | Classification_binary: 0.7926 | Regression_cos_phi: 0.0678 | Regression_energy: 0.4126 | Regression_eta: 0.0639 | Regression_pt: 0.5675 | Regression_sin_phi: 0.0392 | LR: 7.19e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7122s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2064 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2822.7 | uss_MB: 1580.3 | child_rss_MB: 22724.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:40:13,016] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:40:13,063] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:40:15,525] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:40:15,525] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:56,324] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:40:56,324] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:25,583] INFO : Step 33000/500000 rank0 | Train Loss: 1.5106 | Train Loss Unweighted: 1.0012 | Train Loss Components: Classification: 0.0022 | Classification_binary: 0.9252 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0301 | Regression_eta: 0.0046 | Regression_pt: 0.0310 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0277 | Classification_binary: 0.9252 | Regression_cos_phi: 0.0282 | Regression_energy: 0.1807 | Regression_eta: 0.0289 | Regression_pt: 0.2761 | Regression_sin_phi: 0.0439 | LR: 7.51e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7123s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0251 | max_optimizer_step: 0.0777 | max_tensorboard_logging: 0.2193 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2728.9 | uss_MB: 1486.5 | child_rss_MB: 22723.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:41:36,777] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 04:41:36,777] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:17,344] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 04:42:17,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:38,205] INFO : Step 34000/500000 rank0 | Train Loss: 1.8048 | Train Loss Unweighted: 1.2016 | Train Loss Components: Classification: 0.0016 | Classification_binary: 1.1100 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0370 | Regression_eta: 0.0076 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0195 | Classification_binary: 1.1100 | Regression_cos_phi: 0.0349 | Regression_energy: 0.2221 | Regression_eta: 0.0479 | Regression_pt: 0.3467 | Regression_sin_phi: 0.0238 | LR: 7.83e-05 | DataLoad Time: 0.0136s | Model Forward Time: 0.6943s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2005 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2729.4 | uss_MB: 1487.0 | child_rss_MB: 22727.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:42:38,207] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 04:42:38,254] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:42:58,585] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 04:42:58,585] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:39,003] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 04:43:39,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:51,310] INFO : Step 35000/500000 rank0 | Train Loss: 1.7736 | Train Loss Unweighted: 1.1257 | Train Loss Components: Classification: 0.0007 | Classification_binary: 1.0265 | Regression_cos_phi: 0.0068 | Regression_energy: 0.0411 | Regression_eta: 0.0073 | Regression_pt: 0.0355 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0089 | Classification_binary: 1.0265 | Regression_cos_phi: 0.0598 | Regression_energy: 0.2468 | Regression_eta: 0.0460 | Regression_pt: 0.3166 | Regression_sin_phi: 0.0690 | LR: 8.16e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7517s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.1097 | max_tensorboard_logging: 0.2137 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2732.4 | uss_MB: 1490.0 | child_rss_MB: 22726.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:44:20,002] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 04:44:20,002] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:59,305] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 04:44:59,306] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:02,197] INFO : Step 36000/500000 rank0 | Train Loss: 0.8087 | Train Loss Unweighted: 0.3775 | Train Loss Components: Classification: 0.0045 | Classification_binary: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0204 | Regression_eta: 0.0062 | Regression_pt: 0.0257 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.3155 | Regression_cos_phi: 0.0271 | Regression_energy: 0.1226 | Regression_eta: 0.0390 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0196 | LR: 8.50e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7060s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0033 | max_backward: 0.0255 | max_optimizer_step: 0.0786 | max_tensorboard_logging: 0.2055 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2732.4 | uss_MB: 1490.0 | child_rss_MB: 22726.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:45:02,199] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 04:45:02,245] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:45:39,369] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 04:45:39,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:13,738] INFO : Step 37000/500000 rank0 | Train Loss: 2.5341 | Train Loss Unweighted: 1.6463 | Train Loss Components: Classification: 0.0032 | Classification_binary: 1.5142 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0505 | Regression_eta: 0.0081 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 1.5142 | Regression_cos_phi: 0.0322 | Regression_energy: 0.3035 | Regression_eta: 0.0514 | Regression_pt: 0.5007 | Regression_sin_phi: 0.0924 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7078s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0781 | max_tensorboard_logging: 0.2103 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2732.7 | uss_MB: 1490.3 | child_rss_MB: 22725.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:46:20,356] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 04:46:20,357] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:00,927] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 04:47:00,928] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:26,549] INFO : Step 38000/500000 rank0 | Train Loss: 1.2886 | Train Loss Unweighted: 0.6504 | Train Loss Components: Classification: 0.0085 | Classification_binary: 0.5603 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0283 | Regression_eta: 0.0084 | Regression_pt: 0.0351 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1056 | Classification_binary: 0.5603 | Regression_cos_phi: 0.0425 | Regression_energy: 0.1698 | Regression_eta: 0.0533 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0444 | LR: 9.21e-05 | DataLoad Time: 0.0091s | Model Forward Time: 0.7071s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0791 | max_tensorboard_logging: 0.2090 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2729.7 | uss_MB: 1487.3 | child_rss_MB: 22726.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:47:26,551] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 04:47:26,597] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:47:42,153] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 04:47:42,153] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:22,642] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 04:48:22,642] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:39,299] INFO : Step 39000/500000 rank0 | Train Loss: 1.7061 | Train Loss Unweighted: 1.0133 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.9081 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0388 | Regression_eta: 0.0110 | Regression_pt: 0.0406 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0079 | Classification_binary: 0.9081 | Regression_cos_phi: 0.0650 | Regression_energy: 0.2332 | Regression_eta: 0.0695 | Regression_pt: 0.3616 | Regression_sin_phi: 0.0608 | LR: 9.57e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7186s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0232 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.0804 | max_tensorboard_logging: 0.2130 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2732.4 | uss_MB: 1490.0 | child_rss_MB: 22725.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:49:03,762] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 04:49:03,763] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:44,188] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 04:49:44,188] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:51,857] INFO : Step 40000/500000 rank0 | Train Loss: 0.9758 | Train Loss Unweighted: 0.3811 | Train Loss Components: Classification: 0.0052 | Classification_binary: 0.2926 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0382 | Regression_eta: 0.0047 | Regression_pt: 0.0349 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.2926 | Regression_cos_phi: 0.0254 | Regression_energy: 0.2293 | Regression_eta: 0.0297 | Regression_pt: 0.3113 | Regression_sin_phi: 0.0235 | LR: 9.94e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7025s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2123 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2732.8 | uss_MB: 1490.4 | child_rss_MB: 22724.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:49:51,859] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 04:49:51,906] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 04:49:51,907] INFO : Running validation on rank0 +rank0 [2026-11-14 04:49:51,907] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:49:51,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:49:51,987] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 04:49:52,061] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:49:52,129] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:49:52,177] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:49:52,222] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:49:52,267] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:49:52,313] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:49:52,359] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:49:52,431] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:49:52,500] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:49:52,567] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:49:52,613] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:49:52,658] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:49:52,703] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:49:52,749] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:49:52,795] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:49:52,867] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:49:52,871] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:49:52,873] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:49:52,874] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:49:52,876] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:49:52,877] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:49:52,879] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:49:52,880] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:49:52,882] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:49:52,883] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:49:52,885] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:49:52,886] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:49:52,887] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:49:52,888] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:49:52,889] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:49:52,891] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:49:53,823] INFO : VALIDATION | Step=40000/500000 | Train Loss=0.9758 | Valid Loss=10.4008 | Train Loss Unweighted=0.3811 | Valid Loss Unweighted=5.2852 | Stale=2 | ETA=559.4m +rank0 [2026-11-14 04:49:53,824] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:49:53,921] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:49:53,930] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:49:53,974] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:49:54,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:49:54,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:49:54,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:49:54,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:49:54,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:49:55,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:49:55,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:49:55,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:49:55,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:49:55,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:49:55,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:49:56,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:49:56,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:49:56,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:49:56,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:49:56,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:49:56,739] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:49:56,747] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:49:57,101] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:49:57,369] INFO : Plotted number of elements +rank0 [2026-11-14 04:49:58,251] INFO : Plotted jets +rank0 [2026-11-14 04:49:58,920] INFO : Plotted jet ratio +rank0 [2026-11-14 04:50:32,618] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 04:50:32,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:11,207] INFO : Step 41000/500000 rank0 | Train Loss: 1.3400 | Train Loss Unweighted: 0.6179 | Train Loss Components: Classification: 0.0015 | Classification_binary: 0.5080 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0495 | Regression_eta: 0.0031 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0183 | Classification_binary: 0.5080 | Regression_cos_phi: 0.0272 | Regression_energy: 0.2976 | Regression_eta: 0.0197 | Regression_pt: 0.4525 | Regression_sin_phi: 0.0166 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7273s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0298 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2179 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2756.6 | uss_MB: 1514.2 | child_rss_MB: 22724.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:51:13,019] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 04:51:13,019] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:53,158] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 04:51:53,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:23,493] INFO : Step 42000/500000 rank0 | Train Loss: 2.0359 | Train Loss Unweighted: 0.9379 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.7689 | Regression_cos_phi: 0.0036 | Regression_energy: 0.0770 | Regression_eta: 0.0066 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0096 | Classification_binary: 0.7689 | Regression_cos_phi: 0.0318 | Regression_energy: 0.4629 | Regression_eta: 0.0417 | Regression_pt: 0.7010 | Regression_sin_phi: 0.0200 | LR: 1.07e-04 | DataLoad Time: 0.0158s | Model Forward Time: 0.7118s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0268 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.2109 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2757.1 | uss_MB: 1514.8 | child_rss_MB: 22727.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:52:23,494] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 04:52:23,543] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 04:52:34,375] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 04:52:34,375] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:15,327] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 04:53:15,327] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:36,821] INFO : Step 43000/500000 rank0 | Train Loss: 1.4576 | Train Loss Unweighted: 0.5827 | Train Loss Components: Classification: 0.0005 | Classification_binary: 0.4478 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0629 | Regression_eta: 0.0040 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0061 | Classification_binary: 0.4478 | Regression_cos_phi: 0.0250 | Regression_energy: 0.3779 | Regression_eta: 0.0253 | Regression_pt: 0.5380 | Regression_sin_phi: 0.0375 | LR: 1.11e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7360s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0235 | max_loss: 0.0022 | max_backward: 0.0261 | max_optimizer_step: 0.1054 | max_tensorboard_logging: 0.2115 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2760.2 | uss_MB: 1517.8 | child_rss_MB: 22726.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:53:56,582] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 04:53:56,582] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:37,243] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 04:54:37,243] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:49,521] INFO : Step 44000/500000 rank0 | Train Loss: 0.8233 | Train Loss Unweighted: 0.3113 | Train Loss Components: Classification: 0.0011 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0066 | Regression_energy: 0.0302 | Regression_eta: 0.0056 | Regression_pt: 0.0307 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0137 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0584 | Regression_energy: 0.1814 | Regression_eta: 0.0355 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0266 | LR: 1.15e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7045s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0264 | max_loss: 0.0023 | max_backward: 0.0259 | max_optimizer_step: 0.0785 | max_tensorboard_logging: 0.2039 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2760.6 | uss_MB: 1518.2 | child_rss_MB: 22726.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:54:49,524] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 04:54:49,572] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 04:55:18,329] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 04:55:18,329] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:55:59,132] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 04:55:59,132] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:02,672] INFO : Step 45000/500000 rank0 | Train Loss: 1.2738 | Train Loss Unweighted: 0.2477 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.0883 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0778 | Regression_eta: 0.0038 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0017 | Classification_binary: 0.0883 | Regression_cos_phi: 0.0168 | Regression_energy: 0.4673 | Regression_eta: 0.0239 | Regression_pt: 0.6561 | Regression_sin_phi: 0.0198 | LR: 1.19e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7184s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0022 | max_backward: 0.0264 | max_optimizer_step: 0.0756 | max_tensorboard_logging: 0.2100 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2760.6 | uss_MB: 1518.2 | child_rss_MB: 22726.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:56:40,183] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 04:56:40,183] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:57:14,418] INFO : Step 46000/500000 rank0 | Train Loss: 0.8477 | Train Loss Unweighted: 0.1518 | Train Loss Components: Classification: 0.0093 | Classification_binary: 0.0505 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0407 | Regression_eta: 0.0076 | Regression_pt: 0.0376 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.1159 | Classification_binary: 0.0505 | Regression_cos_phi: 0.0249 | Regression_energy: 0.2446 | Regression_eta: 0.0480 | Regression_pt: 0.3352 | Regression_sin_phi: 0.0286 | LR: 1.23e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7134s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0234 | max_loss: 0.0022 | max_backward: 0.0253 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2757.2 | uss_MB: 1514.8 | child_rss_MB: 22725.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:57:14,420] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 04:57:14,469] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 04:57:20,338] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 04:57:20,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:00,780] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 04:58:00,780] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:26,887] INFO : Step 47000/500000 rank0 | Train Loss: 1.2191 | Train Loss Unweighted: 0.4862 | Train Loss Components: Classification: 0.0012 | Classification_binary: 0.3746 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0491 | Regression_eta: 0.0042 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0146 | Classification_binary: 0.3746 | Regression_cos_phi: 0.0374 | Regression_energy: 0.2950 | Regression_eta: 0.0264 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0253 | LR: 1.27e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7269s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0263 | max_optimizer_step: 0.0791 | max_tensorboard_logging: 0.2106 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2760.2 | uss_MB: 1517.9 | child_rss_MB: 22727.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:58:41,818] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 04:58:41,818] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:22,231] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 04:59:22,231] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:39,439] INFO : Step 48000/500000 rank0 | Train Loss: 0.6138 | Train Loss Unweighted: 0.2734 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.2211 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0205 | Regression_eta: 0.0052 | Regression_pt: 0.0217 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0021 | Classification_binary: 0.2211 | Regression_cos_phi: 0.0272 | Regression_energy: 0.1230 | Regression_eta: 0.0330 | Regression_pt: 0.1937 | Regression_sin_phi: 0.0138 | LR: 1.31e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7166s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0251 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2760.3 | uss_MB: 1517.9 | child_rss_MB: 22725.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:59:39,440] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 04:59:39,486] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:00:03,012] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:00:03,013] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:43,433] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:00:43,433] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:51,710] INFO : Step 49000/500000 rank0 | Train Loss: 2.7575 | Train Loss Unweighted: 0.7737 | Train Loss Components: Classification: 0.0013 | Classification_binary: 0.4690 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1420 | Regression_eta: 0.0068 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0158 | Classification_binary: 0.4690 | Regression_cos_phi: 0.0698 | Regression_energy: 0.8533 | Regression_eta: 0.0428 | Regression_pt: 1.2726 | Regression_sin_phi: 0.0344 | LR: 1.36e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7138s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0787 | max_tensorboard_logging: 0.2155 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2760.5 | uss_MB: 1518.2 | child_rss_MB: 22724.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:01:24,574] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:01:24,575] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:04,592] INFO : Step 50000/500000 rank0 | Train Loss: 1.5569 | Train Loss Unweighted: 0.8630 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.7554 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0494 | Regression_eta: 0.0056 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 12.4250 | Classification_binary: 1.0000 | Regression_cos_phi: 8.7946 | Regression_energy: 6.0087 | Regression_eta: 6.3205 | Regression_pt: 8.9097 | Regression_sin_phi: 8.8634 | Task Weighted Losses: Classification: 0.0035 | Classification_binary: 0.7554 | Regression_cos_phi: 0.0177 | Regression_energy: 0.2966 | Regression_eta: 0.0352 | Regression_pt: 0.4174 | Regression_sin_phi: 0.0310 | LR: 1.40e-04 | DataLoad Time: 0.0052s | Model Forward Time: 0.7122s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0262 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2756.8 | uss_MB: 1514.4 | child_rss_MB: 22726.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:02:04,594] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:02:04,641] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:02:04,641] INFO : Running validation on rank0 +rank0 [2026-11-14 05:02:04,642] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:02:04,642] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:02:04,723] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:02:04,796] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:02:04,863] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:02:04,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:02:04,958] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:02:05,004] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:02:05,050] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:02:05,095] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:02:05,169] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:02:05,238] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:02:05,306] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:02:05,352] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:02:05,397] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:02:05,442] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:02:05,489] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:02:05,533] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:02:05,598] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:02:05,602] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:02:05,603] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:02:05,605] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:02:05,606] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:02:05,608] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:02:05,609] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:02:05,611] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:02:05,612] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:02:05,614] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:02:05,615] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:02:05,616] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:02:05,618] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:02:05,618] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:02:05,620] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:02:05,621] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:02:06,516] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.5569 | Valid Loss=11.4547 | Train Loss Unweighted=0.8630 | Valid Loss Unweighted=6.3916 | Stale=3 | ETA=547.7m +rank0 [2026-11-14 05:02:06,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:02:06,612] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:02:06,620] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:02:06,663] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:02:07,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:02:07,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:02:07,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:02:07,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:02:07,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:02:07,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:02:07,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:02:08,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:02:08,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:02:08,443] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:02:08,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:02:08,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:02:08,895] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:02:09,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:02:09,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:02:09,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:02:09,411] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 05:02:09,417] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:02:09,780] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_3mm_seed12345_20261114_040046_838903/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:02:10,047] INFO : Plotted number of elements +rank0 [2026-11-14 05:02:10,921] INFO : Plotted jets +rank0 [2026-11-14 05:02:11,582] INFO : Plotted jet ratio +rank0 [2026-11-14 05:02:12,037] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:02:12,037] INFO : Training completed. Total time on device 0: 60.951min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..e9317bcb1ca6ba76a002492325bc65761f8a22dd --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d21fe5e6abe3619ff33274709458c7cbde7ae9b5776a4381576ede0c399ab73d +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-50000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..4a817c6d3a4d2e6b31e7027e80ec50bd6dc21f5e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be6cdc3debf216e75555eac7e2158e7f6db2db7ecc56e2d9b020f655b395d320 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..50fd85081245e19df3cbf3f9ce07e42a2660e613 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 3.072474241256714, "Classification": 0.034949954599142075, "Regression_pt": 0.23677295446395874, "Regression_eta": 0.010671804659068584, "Regression_sin_phi": 0.018434815108776093, "Regression_cos_phi": 0.011741811409592628, "Regression_energy": 0.25216934084892273, "Total": 7.684525489807129, "Total_unweighted": 3.6372149223461747}, "train_group": {}, "valid": {"Classification_binary": 2.3794960975646973, "Classification": 0.019277988001704216, "Regression_pt": 0.2195357233285904, "Regression_eta": 0.033894289284944534, "Regression_sin_phi": 0.011582530103623867, "Regression_cos_phi": 0.01441083662211895, "Regression_energy": 0.2210903912782669, "Total": 6.542428016662598, "Total_unweighted": 2.899287856183946, "metrics/particle/class_1/efficiency": 0.8538283109664917, "metrics/particle/class_1/pid_accuracy": 0.9728260636329651, "metrics/particle/class_2/efficiency": 0.17391304671764374, "metrics/particle/class_2/pid_accuracy": 0.75, "metrics/particle/class_3/efficiency": 0.6150793433189392, "metrics/particle/class_3/pid_accuracy": 0.977419376373291, "metrics/particle/class_4/efficiency": 0.8921568393707275, "metrics/particle/class_4/pid_accuracy": 0.901098906993866, "metrics/particle/class_5/efficiency": 0.8723404407501221, "metrics/particle/class_5/pid_accuracy": 0.9268292784690857, "metrics/particle/count/bias_mean": -0.15199999511241913, "metrics/particle/count/mae": 0.6959999799728394, "metrics/particle/count/prediction_mean": 4.464000225067139, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.09076806157827377, "metrics/particle/event/energy_response_mean": 0.972862184047699, "metrics/particle/event/met_abs_error": 2.396327018737793, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09309843927621841, "metrics/particle/event/scalar_pt_response_mean": 0.971147358417511, "metrics/particle/event/vector_pt_closure": 0.09935760498046875, "metrics/particle/matched/delta_eta_abs_mean": 0.029710207134485245, "metrics/particle/matched/delta_phi_abs_mean": 0.024580100551247597, "metrics/particle/matched/delta_r_mean": 0.04290895164012909, "metrics/particle/matched/energy_relative_abs_mean": 0.08439955860376358, "metrics/particle/matched/pid_accuracy": 0.961403489112854, "metrics/particle/matched/pt_relative_abs_mean": 0.08227109909057617, "metrics/particle/matching/duplicate_fraction": 0.018817204982042313, "metrics/particle/matching/efficiency": 0.7409012317657471, "metrics/particle/matching/f1": 0.7533039450645447, "metrics/particle/matching/purity": 0.7661290168762207, "metrics/particle/matching/target_coverage_dr0p05": 0.49046793580055237, "metrics/particle/matching/target_coverage_dr0p10": 0.7634315490722656, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7287694811820984, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8344887495040894}, "step_train_time": 0.7464909553527832, "step_valid_time": 1.7257616720162332, "step_total_time": 732.8791680335999, "elapsed_training_loop_seconds": 732.8791680335999, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9910649889182329, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08964531523906416, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 401, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 394, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9303944315545244, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9391100702576113, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9347319347319347, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06088992974238876, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.91415313225058, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9227166276346604, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9184149184149184, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9303944315545244, "step_test_time": 2.9383501389529556, "step_plot_time": 2.2834928130032495, "step_validation_cycle_time": 7.429154915036634} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..37168dae33290d50547ba264ba7f5077ba680a5f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 2.381514549255371, "Classification": 0.012629280798137188, "Regression_pt": 0.08552266657352448, "Regression_eta": 0.06692903488874435, "Regression_sin_phi": 0.00900278054177761, "Regression_cos_phi": 0.008567686192691326, "Regression_energy": 0.08583245426416397, "Total": 4.459038257598877, "Total_unweighted": 2.64999845251441}, "train_group": {}, "valid": {"Classification_binary": 2.3622515201568604, "Classification": 0.017761386930942535, "Regression_pt": 0.22645148634910583, "Regression_eta": 0.029404040426015854, "Regression_sin_phi": 0.009599070996046066, "Regression_cos_phi": 0.009354195557534695, "Regression_energy": 0.22470040619373322, "Total": 6.504622936248779, "Total_unweighted": 2.8795221066102386, "metrics/particle/class_1/efficiency": 0.8909512758255005, "metrics/particle/class_1/pid_accuracy": 0.9661458134651184, "metrics/particle/class_2/efficiency": 0.1304347813129425, "metrics/particle/class_2/pid_accuracy": 1.0, "metrics/particle/class_3/efficiency": 0.5992063283920288, "metrics/particle/class_3/pid_accuracy": 0.9768211841583252, "metrics/particle/class_4/efficiency": 0.9313725233078003, "metrics/particle/class_4/pid_accuracy": 0.9263157844543457, "metrics/particle/class_5/efficiency": 0.9680851101875305, "metrics/particle/class_5/pid_accuracy": 0.9230769276618958, "metrics/particle/count/bias_mean": -0.2160000056028366, "metrics/particle/count/mae": 0.6159999966621399, "metrics/particle/count/prediction_mean": 4.400000095367432, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.07206004112958908, "metrics/particle/event/energy_response_mean": 0.985042929649353, "metrics/particle/event/met_abs_error": 2.0956838130950928, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08282063901424408, "metrics/particle/event/scalar_pt_response_mean": 0.9942255616188049, "metrics/particle/event/vector_pt_closure": 0.09195639938116074, "metrics/particle/matched/delta_eta_abs_mean": 0.024351119995117188, "metrics/particle/matched/delta_phi_abs_mean": 0.0217114370316267, "metrics/particle/matched/delta_r_mean": 0.03611977398395538, "metrics/particle/matched/energy_relative_abs_mean": 0.0793398767709732, "metrics/particle/matched/pid_accuracy": 0.9611428380012512, "metrics/particle/matched/pt_relative_abs_mean": 0.07977067679166794, "metrics/particle/matching/duplicate_fraction": 0.019999999552965164, "metrics/particle/matching/efficiency": 0.7582322359085083, "metrics/particle/matching/f1": 0.7763975262641907, "metrics/particle/matching/purity": 0.7954545617103577, "metrics/particle/matching/target_coverage_dr0p05": 0.5935875177383423, "metrics/particle/matching/target_coverage_dr0p10": 0.7842287421226501, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7409012317657471, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8388214707374573}, "step_train_time": 0.8038413524627686, "step_valid_time": 1.905121897929348, "step_total_time": 1473.407804965973, "elapsed_training_loop_seconds": 1473.407804965973, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9967591437394727, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08123406123420862, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 428, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 409, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 406, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9489559164733179, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.955607476635514, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9522700814901047, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04439252336448598, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9419953596287703, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9485981308411215, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9452852153667055, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9489559164733179, "step_test_time": 2.976234958972782, "step_plot_time": 2.16242539009545, "step_validation_cycle_time": 7.5674342579441145} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..60bd355c310fac1c2c35057353f0e502462f60e7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 1.1132192611694336, "Classification": 0.0033858073875308037, "Regression_pt": 0.02879033237695694, "Regression_eta": 0.007109193131327629, "Regression_sin_phi": 0.0037444306071847677, "Regression_cos_phi": 0.004998145159333944, "Regression_energy": 0.027381034567952156, "Total": 1.7241840362548828, "Total_unweighted": 1.1886282043997198}, "train_group": {}, "valid": {"Classification_binary": 2.760695457458496, "Classification": 0.02088908664882183, "Regression_pt": 0.26017099618911743, "Regression_eta": 0.02778015471994877, "Regression_sin_phi": 0.009523475542664528, "Regression_cos_phi": 0.008775264024734497, "Regression_energy": 0.2578212320804596, "Total": 7.45579719543457, "Total_unweighted": 3.3456556666642427, "metrics/particle/class_1/efficiency": 0.895591676235199, "metrics/particle/class_1/pid_accuracy": 0.9637305736541748, "metrics/particle/class_2/efficiency": 0.08695652335882187, "metrics/particle/class_2/pid_accuracy": 1.0, "metrics/particle/class_3/efficiency": 0.6230158805847168, "metrics/particle/class_3/pid_accuracy": 0.9808917045593262, "metrics/particle/class_4/efficiency": 0.9509803652763367, "metrics/particle/class_4/pid_accuracy": 0.9278350472450256, "metrics/particle/class_5/efficiency": 0.9893617033958435, "metrics/particle/class_5/pid_accuracy": 0.9354838728904724, "metrics/particle/count/bias_mean": -0.14000000059604645, "metrics/particle/count/mae": 0.6200000047683716, "metrics/particle/count/prediction_mean": 4.47599983215332, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.09408905357122421, "metrics/particle/event/energy_response_mean": 1.022140622138977, "metrics/particle/event/met_abs_error": 2.2674567699432373, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0986265316605568, "metrics/particle/event/scalar_pt_response_mean": 1.027095913887024, "metrics/particle/event/vector_pt_closure": 0.09824547916650772, "metrics/particle/matched/delta_eta_abs_mean": 0.02478361688554287, "metrics/particle/matched/delta_phi_abs_mean": 0.0209977850317955, "metrics/particle/matched/delta_r_mean": 0.03620826080441475, "metrics/particle/matched/energy_relative_abs_mean": 0.09852316975593567, "metrics/particle/matched/pid_accuracy": 0.963004469871521, "metrics/particle/matched/pt_relative_abs_mean": 0.09977136552333832, "metrics/particle/matching/duplicate_fraction": 0.01787310093641281, "metrics/particle/matching/efficiency": 0.7729635834693909, "metrics/particle/matching/f1": 0.7848657965660095, "metrics/particle/matching/purity": 0.7971403002738953, "metrics/particle/matching/target_coverage_dr0p05": 0.5996533632278442, "metrics/particle/matching/target_coverage_dr0p10": 0.7911611795425415, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7123050093650818, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8466204404830933}, "step_train_time": 0.745375394821167, "step_valid_time": 1.9087854019599035, "step_total_time": 2220.0213556289673, "elapsed_training_loop_seconds": 2220.0213556289673, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0487089940128396, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07804283622807096, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 413, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 409, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9582366589327146, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9672131147540983, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9627039627039627, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03278688524590164, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9489559164733179, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9578454332552693, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9533799533799534, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9582366589327146, "step_test_time": 3.0076284609967843, "step_plot_time": 2.153984465985559, "step_validation_cycle_time": 7.530620607081801} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..16883d36f0a0b3889d7db2b72106e818b3ae1567 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 0.18598636984825134, "Classification": 0.0005903308046981692, "Regression_pt": 0.02794242836534977, "Regression_eta": 0.007840363308787346, "Regression_sin_phi": 0.006307527422904968, "Regression_cos_phi": 0.005954771302640438, "Regression_energy": 0.027292195707559586, "Total": 0.7901319265365601, "Total_unweighted": 0.2619139867601916}, "train_group": {}, "valid": {"Classification_binary": 4.301364421844482, "Classification": 0.028233123943209648, "Regression_pt": 0.24804411828517914, "Regression_eta": 0.0293711069971323, "Regression_sin_phi": 0.01033164095133543, "Regression_cos_phi": 0.008330370299518108, "Regression_energy": 0.2500151991844177, "Total": 8.935032844543457, "Total_unweighted": 4.875689981505275, "metrics/particle/class_1/efficiency": 0.8932714462280273, "metrics/particle/class_1/pid_accuracy": 0.9558441638946533, "metrics/particle/class_2/efficiency": 0.1304347813129425, "metrics/particle/class_2/pid_accuracy": 0.3333333432674408, "metrics/particle/class_3/efficiency": 0.6111111044883728, "metrics/particle/class_3/pid_accuracy": 0.9707792401313782, "metrics/particle/class_4/efficiency": 0.9215686321258545, "metrics/particle/class_4/pid_accuracy": 0.9042553305625916, "metrics/particle/class_5/efficiency": 0.978723406791687, "metrics/particle/class_5/pid_accuracy": 0.9239130616188049, "metrics/particle/count/bias_mean": -0.012000000104308128, "metrics/particle/count/mae": 0.5879999995231628, "metrics/particle/count/prediction_mean": 4.604000091552734, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.07799656689167023, "metrics/particle/event/energy_response_mean": 1.005791425704956, "metrics/particle/event/met_abs_error": 2.1267030239105225, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08105529844760895, "metrics/particle/event/scalar_pt_response_mean": 1.00830078125, "metrics/particle/event/vector_pt_closure": 0.09194205701351166, "metrics/particle/matched/delta_eta_abs_mean": 0.02443300187587738, "metrics/particle/matched/delta_phi_abs_mean": 0.020731447264552116, "metrics/particle/matched/delta_r_mean": 0.03540976718068123, "metrics/particle/matched/energy_relative_abs_mean": 0.08175250142812729, "metrics/particle/matched/pid_accuracy": 0.9501133561134338, "metrics/particle/matched/pt_relative_abs_mean": 0.07955627143383026, "metrics/particle/matching/duplicate_fraction": 0.023457862436771393, "metrics/particle/matching/efficiency": 0.7642980813980103, "metrics/particle/matching/f1": 0.7652928233146667, "metrics/particle/matching/purity": 0.7662901878356934, "metrics/particle/matching/target_coverage_dr0p05": 0.5918543934822083, "metrics/particle/matching/target_coverage_dr0p10": 0.7885615229606628, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7348353266716003, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8457539081573486}, "step_train_time": 0.766629695892334, "step_valid_time": 1.940529576037079, "step_total_time": 2960.7437801361084, "elapsed_training_loop_seconds": 2960.7437801361084, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.017956405613307, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05774221559735482, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 406, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 398, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9419953596287703, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9508196721311475, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9463869463869464, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04918032786885246, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9234338747099768, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9320843091334895, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9277389277389277, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9419953596287703, "step_test_time": 2.977500611101277, "step_plot_time": 2.1721612520050257, "step_validation_cycle_time": 7.542139297933318} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..334c84e1a90d95533085060a445f9c04abf20541 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 0.5738043189048767, "Classification": 0.0016867187805473804, "Regression_pt": 0.06925219297409058, "Regression_eta": 0.005786131136119366, "Regression_sin_phi": 0.003433453617617488, "Regression_cos_phi": 0.006621260661631823, "Regression_energy": 0.07244902104139328, "Total": 1.8362517356872559, "Total_unweighted": 0.7330330971162766}, "train_group": {}, "valid": {"Classification_binary": 6.11185359954834, "Classification": 0.04292648658156395, "Regression_pt": 0.29538461565971375, "Regression_eta": 0.02555800788104534, "Regression_sin_phi": 0.009391075000166893, "Regression_cos_phi": 0.008024616166949272, "Regression_energy": 0.29529640078544617, "Total": 11.62879753112793, "Total_unweighted": 6.788434801623225, "metrics/particle/class_1/efficiency": 0.9396751523017883, "metrics/particle/class_1/pid_accuracy": 0.9456790089607239, "metrics/particle/class_2/efficiency": 0.17391304671764374, "metrics/particle/class_2/pid_accuracy": 0.5, "metrics/particle/class_3/efficiency": 0.6527777910232544, "metrics/particle/class_3/pid_accuracy": 0.978723406791687, "metrics/particle/class_4/efficiency": 0.9411764740943909, "metrics/particle/class_4/pid_accuracy": 0.9270833134651184, "metrics/particle/class_5/efficiency": 0.978723406791687, "metrics/particle/class_5/pid_accuracy": 0.9239130616188049, "metrics/particle/count/bias_mean": 0.15199999511241913, "metrics/particle/count/mae": 0.5920000076293945, "metrics/particle/count/prediction_mean": 4.76800012588501, "metrics/particle/count/target_mean": 4.616000175476074, "metrics/particle/event/energy_relative_abs_error": 0.07520198822021484, "metrics/particle/event/energy_response_mean": 1.0068330764770508, "metrics/particle/event/met_abs_error": 2.088735580444336, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07609887421131134, "metrics/particle/event/scalar_pt_response_mean": 1.0159026384353638, "metrics/particle/event/vector_pt_closure": 0.08835471421480179, "metrics/particle/matched/delta_eta_abs_mean": 0.02544061280786991, "metrics/particle/matched/delta_phi_abs_mean": 0.01912212371826172, "metrics/particle/matched/delta_r_mean": 0.03554726019501686, "metrics/particle/matched/energy_relative_abs_mean": 0.07663226872682571, "metrics/particle/matched/pid_accuracy": 0.9514039158821106, "metrics/particle/matched/pt_relative_abs_mean": 0.07480575144290924, "metrics/particle/matching/duplicate_fraction": 0.026006711646914482, "metrics/particle/matching/efficiency": 0.8024263381958008, "metrics/particle/matching/f1": 0.7894288301467896, "metrics/particle/matching/purity": 0.7768456339836121, "metrics/particle/matching/target_coverage_dr0p05": 0.6187174916267395, "metrics/particle/matching/target_coverage_dr0p10": 0.8249566555023193, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.760831892490387, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8734835386276245}, "step_train_time": 0.7506992816925049, "step_valid_time": 1.9154713820898905, "step_total_time": 3704.8876433372498, "elapsed_training_loop_seconds": 3704.8876433372498, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0045309389684578, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.04953787340513427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 409, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 404, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9489559164733179, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9511627906976744, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9500580720092915, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04883720930232558, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9373549883990719, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9395348837209302, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9384436701509872, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9489559164733179, "step_test_time": 2.987430287990719, "step_plot_time": 2.21508956910111, "step_validation_cycle_time": 8.222470176988281} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..78b6be9227f245679cfa6d93a506534e14d8d9ed --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_z_tautau_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..4f9fcd30bbf8de51aa02dc4a4b384e92a89c1147 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9112270c41ebbb6b81d07abf1e9e800f74e036a992a0b1de9ca34d53e921cea2 +size 4402 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..107f7778e0cc05f5c6c7245970a880ba1918effb Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..cc2a600b8b1b457bc3757566e1056b185958f30b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3445a1f5b1c74c9944357251ac838dee6fb9f298 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..85f1d10ca268e5e66e748fa4c3fbd9b91bafb2d9 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8de17fc8519c812ae2ac133f4744a097d3f6bde9 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..f3923bb79b4ff2868cca8ef8f216032d4e6afc73 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..cb890b3ea8c02eeb541a60bc3d0b20c3bc6f8ba4 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..b1c8b3b6cc2924be0661f66a32f562f083b08504 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5710b447b522f327ad19c8128e8fc8ee65087d8b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..f45c4689de20d4ea7ac2653de58295973805ffdd Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a3113f43ec75bffb7de95fbbf06eef6240774422 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..8a0c567fa8c69e818cf505c64fe519b40b02c8ad Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/runs/train/events.out.tfevents.1794646874.workergpu215.3189578.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/runs/train/events.out.tfevents.1794646874.workergpu215.3189578.0 new file mode 100644 index 0000000000000000000000000000000000000000..33ceaba853fcb31805ed20d803c5b389910719f2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/runs/train/events.out.tfevents.1794646874.workergpu215.3189578.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:481c0c3d8481a8092585a3a91ad013e45dbe2cf50111d16aaf5fc1ddb66e1f2a +size 1843423 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/runs/valid/events.out.tfevents.1794646874.workergpu215.3189578.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/runs/valid/events.out.tfevents.1794646874.workergpu215.3189578.1 new file mode 100644 index 0000000000000000000000000000000000000000..bffa23af4a389781793b5b6d3d42b37d1b14a86e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/runs/valid/events.out.tfevents.1794646874.workergpu215.3189578.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6722ea230415fec62123f8a555fb6a44a25a2d697c6d4ab0d99dfc3c4bfcb6 +size 2317721 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..35bcae07e592543ec77c27e94f98974938988ccc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_4mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85da626a836ad9e9b52683519cb452129ee11f16 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_4mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c1973964259e50a1e6b7fe7bdb45f6a0a9b8de89 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/train.log @@ -0,0 +1,999 @@ +rank0 [2026-11-14 04:01:11,084] INFO : Will use single-gpu: NVIDIA H100 +rank0 [2026-11-14 04:01:11,084] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 04:01:12,417] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 04:01:12,417] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 04:01:12,417] INFO : Instantiating model +rank0 [2026-11-14 04:01:12,435] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 04:01:12,435] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 04:01:12,437] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 04:01:12,437] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 04:01:12,437] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 04:01:12,437] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 04:01:12,437] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,439] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 04:01:12,439] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,441] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 04:01:12,441] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,444] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 04:01:12,444] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,446] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 04:01:12,446] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,448] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 04:01:12,448] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,451] INFO : Backbone initialization took 0.01s +rank0 [2026-11-14 04:01:12,451] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 04:01:12,451] INFO : Initializing output DNNs +rank0 [2026-11-14 04:01:12,452] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,454] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,460] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 04:01:12,460] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 04:01:12,460] INFO : jagged_attention enabled=True +rank0 [2026-11-14 04:01:12,460] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 04:01:12,460] INFO : task_queries enabled=True +rank0 [2026-11-14 04:01:12,460] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 04:01:12,460] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 04:01:12,461] INFO : classification_query parameters: 256 +rank0 [2026-11-14 04:01:12,461] INFO : regression_query parameters: 256 +rank0 [2026-11-14 04:01:12,461] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 04:01:12,461] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 04:01:12,461] INFO : output_mode=elementwise +rank0 [2026-11-14 04:01:12,461] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 04:01:12,461] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 04:01:12,461] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 04:01:12,461] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 04:01:12,461] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 04:01:12,461] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 04:01:12,461] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 04:01:12,461] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 04:01:12,462] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 04:01:12,462] INFO : MLPF __init__ done +rank0 [2026-11-14 04:01:12,462] INFO : Instantiated model +rank0 [2026-11-14 04:01:12,463] INFO : Moving model to device rank=0 +rank0 [2026-11-14 04:01:12,468] INFO : Moved model to device rank=0 +rank0 [2026-11-14 04:01:12,469] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 04:01:12,537] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 04:01:12,537] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 04:01:12,538] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 04:01:12,538] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 04:01:12,538] INFO : Total parameters: 3910930 +rank0 [2026-11-14 04:01:12,540] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 04:01:12,541] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296 +rank0 [2026-11-14 04:01:12,541] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296 +rank0 [2026-11-14 04:01:12,541] INFO : Getting dataloaders +rank0 [2026-11-14 04:01:12,541] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:01:14,177] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,262] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,328] INFO : train_dataset: cld_edm_z_tautau_pf, 898 +rank0 [2026-11-14 04:01:14,364] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,414] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,459] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,502] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,540] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,586] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,621] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,621] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 04:01:14,622] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 04:01:14,622] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 04:01:14,622] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:01:14,631] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,642] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,651] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,661] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,671] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,681] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,691] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,701] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,710] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,720] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,720] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 04:01:14,721] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 04:01:14,754] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:01:14,955] INFO : Got dataloaders +rank0 [2026-11-14 04:01:14,955] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 04:01:14,955] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 04:01:14,955] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 04:01:14,959] INFO : Creating train_iterator +rank0 [2026-11-14 04:01:14,959] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:14,992] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:01:15,206] INFO : Created train_iterator +rank0 [2026-11-14 04:01:23,354] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.4305033683776855, 'Classification': 0.60180958122015, 'Regression_pt': 0.7710086131095886, 'Regression_eta': 1.221010594367981, 'Regression_sin_phi': 0.8146398048102855, 'Regression_cos_phi': 0.8275578145682811, 'Regression_energy': 1.2010143318772315}, weights={'Classification_binary': 1.0, 'Classification': 12.34693431854248, 'Regression_pt': 9.637380599975586, 'Regression_eta': 6.085535526275635, 'Regression_sin_phi': 9.12121295928955, 'Regression_cos_phi': 8.978833198547363, 'Regression_energy': 6.186856746673584} +rank0 [2026-11-14 04:01:56,610] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 04:01:56,611] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:02:28,538] INFO : Step 1000/500000 rank0 | Train Loss: 11.6954 | Train Loss Unweighted: 5.3087 | Train Loss Components: Classification: 0.0862 | Classification_binary: 4.4440 | Regression_cos_phi: 0.0229 | Regression_energy: 0.3268 | Regression_eta: 0.0453 | Regression_pt: 0.3618 | Regression_sin_phi: 0.0217 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 1.0637 | Classification_binary: 4.4440 | Regression_cos_phi: 0.2056 | Regression_energy: 2.0218 | Regression_eta: 0.2759 | Regression_pt: 3.4868 | Regression_sin_phi: 0.1975 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7577s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0023 | max_backward: 0.0253 | max_optimizer_step: 0.0841 | max_tensorboard_logging: 0.2243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2615.4 | uss_MB: 1360.9 | child_rss_MB: 22799.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:02:37,277] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:02:37,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:17,518] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:03:17,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:40,918] INFO : Step 2000/500000 rank0 | Train Loss: 8.3653 | Train Loss Unweighted: 4.0104 | Train Loss Components: Classification: 0.0523 | Classification_binary: 3.4109 | Regression_cos_phi: 0.0136 | Regression_energy: 0.2462 | Regression_eta: 0.0281 | Regression_pt: 0.2464 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.6461 | Classification_binary: 3.4109 | Regression_cos_phi: 0.1218 | Regression_energy: 1.5230 | Regression_eta: 0.1712 | Regression_pt: 2.3748 | Regression_sin_phi: 0.1175 | LR: 2.02e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7477s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0256 | max_loss: 0.0023 | max_backward: 0.0250 | max_optimizer_step: 0.0821 | max_tensorboard_logging: 0.2202 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2616.1 | uss_MB: 1361.8 | child_rss_MB: 22796.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:03:40,920] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:03:58,438] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:03:58,439] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:39,552] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:04:39,552] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:53,829] INFO : Step 3000/500000 rank0 | Train Loss: 9.0523 | Train Loss Unweighted: 3.9949 | Train Loss Components: Classification: 0.0984 | Classification_binary: 3.3296 | Regression_cos_phi: 0.0134 | Regression_energy: 0.2506 | Regression_eta: 0.0206 | Regression_pt: 0.2667 | Regression_sin_phi: 0.0156 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 1.2143 | Classification_binary: 3.3296 | Regression_cos_phi: 0.1203 | Regression_energy: 1.5504 | Regression_eta: 0.1254 | Regression_pt: 2.5702 | Regression_sin_phi: 0.1420 | LR: 2.05e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7479s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0832 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2620.1 | uss_MB: 1366.1 | child_rss_MB: 22789.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:05:20,761] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:05:20,761] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:01,446] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:06:01,446] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:06,921] INFO : Step 4000/500000 rank0 | Train Loss: 8.2691 | Train Loss Unweighted: 3.6304 | Train Loss Components: Classification: 0.0350 | Classification_binary: 2.9813 | Regression_cos_phi: 0.0210 | Regression_energy: 0.2652 | Regression_eta: 0.0335 | Regression_pt: 0.2658 | Regression_sin_phi: 0.0286 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.4324 | Classification_binary: 2.9813 | Regression_cos_phi: 0.1882 | Regression_energy: 1.6410 | Regression_eta: 0.2039 | Regression_pt: 2.5614 | Regression_sin_phi: 0.2608 | LR: 2.08e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7581s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0024 | max_backward: 0.0262 | max_optimizer_step: 0.0836 | max_tensorboard_logging: 0.2275 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2620.5 | uss_MB: 1366.5 | child_rss_MB: 22790.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:06:06,924] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:06:43,188] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:06:43,188] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:20,910] INFO : Step 5000/500000 rank0 | Train Loss: 8.7609 | Train Loss Unweighted: 4.3335 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.7084 | Regression_cos_phi: 0.0116 | Regression_energy: 0.2736 | Regression_eta: 0.0281 | Regression_pt: 0.2697 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.3882 | Classification_binary: 3.7084 | Regression_cos_phi: 0.1038 | Regression_energy: 1.6927 | Regression_eta: 0.1713 | Regression_pt: 2.5991 | Regression_sin_phi: 0.0975 | LR: 2.13e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7439s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0822 | max_tensorboard_logging: 0.2202 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2618.0 | uss_MB: 1364.0 | child_rss_MB: 22788.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:07:25,150] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:07:25,150] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:05,572] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:08:05,573] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:33,792] INFO : Step 6000/500000 rank0 | Train Loss: 6.7430 | Train Loss Unweighted: 3.1303 | Train Loss Components: Classification: 0.0108 | Classification_binary: 2.6204 | Regression_cos_phi: 0.0125 | Regression_energy: 0.2235 | Regression_eta: 0.0105 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1332 | Classification_binary: 2.6204 | Regression_cos_phi: 0.1123 | Regression_energy: 1.3827 | Regression_eta: 0.0639 | Regression_pt: 2.3604 | Regression_sin_phi: 0.0701 | LR: 2.19e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7434s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0023 | max_backward: 0.0252 | max_optimizer_step: 0.0827 | max_tensorboard_logging: 0.2177 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2618.0 | uss_MB: 1364.0 | child_rss_MB: 22787.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:08:33,793] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:08:46,764] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:08:46,764] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:27,195] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:09:27,195] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:46,457] INFO : Step 7000/500000 rank0 | Train Loss: 11.5347 | Train Loss Unweighted: 4.6921 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.6982 | Regression_cos_phi: 0.0222 | Regression_energy: 0.3933 | Regression_eta: 0.1194 | Regression_pt: 0.4085 | Regression_sin_phi: 0.0257 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.3060 | Classification_binary: 3.6982 | Regression_cos_phi: 0.1991 | Regression_energy: 2.4334 | Regression_eta: 0.7268 | Regression_pt: 3.9366 | Regression_sin_phi: 0.2346 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7422s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0023 | max_backward: 0.0262 | max_optimizer_step: 0.0835 | max_tensorboard_logging: 0.2134 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2619.0 | uss_MB: 1364.9 | child_rss_MB: 22785.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:10:09,144] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:10:09,144] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:49,997] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:10:49,997] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:00,242] INFO : Step 8000/500000 rank0 | Train Loss: 4.5125 | Train Loss Unweighted: 2.6202 | Train Loss Components: Classification: 0.0070 | Classification_binary: 2.3511 | Regression_cos_phi: 0.0033 | Regression_energy: 0.1220 | Regression_eta: 0.0072 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0861 | Classification_binary: 2.3511 | Regression_cos_phi: 0.0298 | Regression_energy: 0.7548 | Regression_eta: 0.0438 | Regression_pt: 1.2082 | Regression_sin_phi: 0.0387 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7434s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0263 | max_optimizer_step: 0.0826 | max_tensorboard_logging: 0.2162 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2619.0 | uss_MB: 1365.0 | child_rss_MB: 22788.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:11:00,243] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:11:00,306] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:11:31,478] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:11:31,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:12:12,345] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:12:12,346] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:12:13,594] INFO : Step 9000/500000 rank0 | Train Loss: 6.3280 | Train Loss Unweighted: 3.4132 | Train Loss Components: Classification: 0.0056 | Classification_binary: 2.9941 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1939 | Regression_eta: 0.0113 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 2.9941 | Regression_cos_phi: 0.0955 | Regression_energy: 1.1997 | Regression_eta: 0.0686 | Regression_pt: 1.8272 | Regression_sin_phi: 0.0741 | LR: 2.43e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7482s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0023 | max_backward: 0.0253 | max_optimizer_step: 0.0828 | max_tensorboard_logging: 0.2210 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2617.0 | uss_MB: 1363.0 | child_rss_MB: 22789.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:12:53,664] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:12:53,664] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:26,042] INFO : Step 10000/500000 rank0 | Train Loss: 7.6845 | Train Loss Unweighted: 3.6372 | Train Loss Components: Classification: 0.0349 | Classification_binary: 3.0725 | Regression_cos_phi: 0.0117 | Regression_energy: 0.2522 | Regression_eta: 0.0107 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0184 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.4315 | Classification_binary: 3.0725 | Regression_cos_phi: 0.1054 | Regression_energy: 1.5601 | Regression_eta: 0.0649 | Regression_pt: 2.2819 | Regression_sin_phi: 0.1681 | LR: 2.52e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7458s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0251 | max_optimizer_step: 0.0818 | max_tensorboard_logging: 0.2200 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2617.6 | uss_MB: 1363.6 | child_rss_MB: 22789.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:13:26,044] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:13:26,104] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:13:26,105] INFO : Running validation on rank0 +rank0 [2026-11-14 04:13:26,105] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:13:26,105] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:13:26,205] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:13:26,308] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:13:26,357] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:13:26,396] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:13:26,435] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:13:26,468] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:13:26,507] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:13:26,547] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:13:26,583] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:13:26,639] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:13:26,698] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:13:26,735] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:13:26,774] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:13:26,806] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:13:26,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:13:26,884] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:13:26,917] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:13:26,919] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:13:26,921] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:13:26,922] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:13:26,924] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:13:26,926] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:13:26,927] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:13:26,929] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:13:26,930] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:13:26,932] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:13:26,932] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:13:26,934] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:13:26,935] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:13:26,937] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:13:26,938] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:13:26,940] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:13:27,885] INFO : VALIDATION | Step=10000/500000 | Train Loss=7.6845 | Valid Loss=6.5424 | Train Loss Unweighted=3.6372 | Valid Loss Unweighted=2.8993 | Stale=0 | ETA=598.6m +rank0 [2026-11-14 04:13:27,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:13:27,986] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:13:27,991] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:28,038] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:13:28,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:13:28,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:13:28,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:13:28,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:13:29,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:13:29,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:13:29,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:13:29,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:13:29,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:13:29,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:13:30,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:13:30,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:13:30,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:13:30,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:13:30,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:13:30,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:13:30,822] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:13:30,829] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:31,179] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:31,520] INFO : Plotted number of elements +rank0 [2026-11-14 04:13:32,398] INFO : Plotted jets +rank0 [2026-11-14 04:13:33,099] INFO : Plotted jet ratio +rank0 [2026-11-14 04:13:41,656] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:13:41,656] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:21,592] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:14:21,592] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:45,242] INFO : Step 11000/500000 rank0 | Train Loss: 3.9926 | Train Loss Unweighted: 2.3634 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.1422 | Regression_cos_phi: 0.0083 | Regression_energy: 0.0880 | Regression_eta: 0.0088 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.2885 | Classification_binary: 2.1422 | Regression_cos_phi: 0.0745 | Regression_energy: 0.5445 | Regression_eta: 0.0538 | Regression_pt: 0.8200 | Regression_sin_phi: 0.0691 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8005s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.2238 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2767.4 | uss_MB: 1520.2 | child_rss_MB: 22790.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:15:02,040] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:15:02,040] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:42,730] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:15:42,730] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:57,883] INFO : Step 12000/500000 rank0 | Train Loss: 3.4987 | Train Loss Unweighted: 2.0849 | Train Loss Components: Classification: 0.0052 | Classification_binary: 1.8857 | Regression_cos_phi: 0.0070 | Regression_energy: 0.0491 | Regression_eta: 0.0392 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0149 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 1.8857 | Regression_cos_phi: 0.0631 | Regression_energy: 0.3038 | Regression_eta: 0.2385 | Regression_pt: 0.8085 | Regression_sin_phi: 0.1355 | LR: 2.75e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7635s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0254 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.2205 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2767.4 | uss_MB: 1520.2 | child_rss_MB: 22790.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:15:57,885] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:15:57,934] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:16:23,923] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:16:23,923] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:17:03,939] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:17:03,939] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:17:09,870] INFO : Step 13000/500000 rank0 | Train Loss: 2.7313 | Train Loss Unweighted: 1.5699 | Train Loss Components: Classification: 0.0060 | Classification_binary: 1.4045 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0723 | Regression_eta: 0.0082 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 1.4045 | Regression_cos_phi: 0.0287 | Regression_energy: 0.4471 | Regression_eta: 0.0501 | Regression_pt: 0.6913 | Regression_sin_phi: 0.0359 | LR: 2.88e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7621s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2183 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2710.5 | uss_MB: 1463.3 | child_rss_MB: 22788.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:17:45,249] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:17:45,250] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:22,683] INFO : Step 14000/500000 rank0 | Train Loss: 6.5522 | Train Loss Unweighted: 2.5690 | Train Loss Components: Classification: 0.0074 | Classification_binary: 2.0008 | Regression_cos_phi: 0.0150 | Regression_energy: 0.2651 | Regression_eta: 0.0048 | Regression_pt: 0.2700 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0913 | Classification_binary: 2.0008 | Regression_cos_phi: 0.1344 | Regression_energy: 1.6402 | Regression_eta: 0.0292 | Regression_pt: 2.6023 | Regression_sin_phi: 0.0540 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7620s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0843 | max_tensorboard_logging: 0.2239 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2711.2 | uss_MB: 1464.0 | child_rss_MB: 22787.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:18:22,685] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:18:22,733] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:18:26,358] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:18:26,358] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:07,561] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:19:07,561] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:36,892] INFO : Step 15000/500000 rank0 | Train Loss: 7.2494 | Train Loss Unweighted: 3.8746 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0069 | Regression_energy: 0.2298 | Regression_eta: 0.0076 | Regression_pt: 0.2127 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1716 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0618 | Regression_energy: 1.4217 | Regression_eta: 0.0465 | Regression_pt: 2.0498 | Regression_sin_phi: 0.1060 | LR: 3.17e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7591s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0265 | max_optimizer_step: 0.0847 | max_tensorboard_logging: 0.2173 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2713.9 | uss_MB: 1466.7 | child_rss_MB: 22786.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:19:49,634] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:19:49,635] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:30,773] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:20:30,773] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:50,615] INFO : Step 16000/500000 rank0 | Train Loss: 4.3721 | Train Loss Unweighted: 2.4909 | Train Loss Components: Classification: 0.0272 | Classification_binary: 2.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.1089 | Regression_eta: 0.0021 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.3363 | Classification_binary: 2.2369 | Regression_cos_phi: 0.0182 | Regression_energy: 0.6738 | Regression_eta: 0.0127 | Regression_pt: 1.0595 | Regression_sin_phi: 0.0347 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7555s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0872 | max_tensorboard_logging: 0.2196 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2713.9 | uss_MB: 1466.7 | child_rss_MB: 22786.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:20:50,617] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:20:50,666] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:21:12,451] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:21:12,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:53,512] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:21:53,513] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:04,466] INFO : Step 17000/500000 rank0 | Train Loss: 3.2942 | Train Loss Unweighted: 2.2404 | Train Loss Components: Classification: 0.0123 | Classification_binary: 2.0962 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0597 | Regression_eta: 0.0044 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1516 | Classification_binary: 2.0962 | Regression_cos_phi: 0.0244 | Regression_energy: 0.3693 | Regression_eta: 0.0267 | Regression_pt: 0.5950 | Regression_sin_phi: 0.0310 | LR: 3.51e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7504s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2089 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.4 | uss_MB: 1463.2 | child_rss_MB: 22786.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:22:35,854] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:22:35,854] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:16,976] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:23:16,976] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:18,851] INFO : Step 18000/500000 rank0 | Train Loss: 2.2629 | Train Loss Unweighted: 1.4820 | Train Loss Components: Classification: 0.0098 | Classification_binary: 1.3755 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0409 | Regression_eta: 0.0056 | Regression_pt: 0.0422 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1210 | Classification_binary: 1.3755 | Regression_cos_phi: 0.0240 | Regression_energy: 0.2529 | Regression_eta: 0.0340 | Regression_pt: 0.4069 | Regression_sin_phi: 0.0487 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2116 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2711.0 | uss_MB: 1463.8 | child_rss_MB: 22791.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:23:18,852] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:23:18,900] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:23:59,092] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:23:59,092] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:33,190] INFO : Step 19000/500000 rank0 | Train Loss: 4.2935 | Train Loss Unweighted: 2.1450 | Train Loss Components: Classification: 0.0341 | Classification_binary: 1.8557 | Regression_cos_phi: 0.0042 | Regression_energy: 0.1192 | Regression_eta: 0.0072 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.4213 | Classification_binary: 1.8557 | Regression_cos_phi: 0.0373 | Regression_energy: 0.7376 | Regression_eta: 0.0439 | Regression_pt: 1.1498 | Regression_sin_phi: 0.0478 | LR: 3.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7815s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0026 | max_backward: 0.0269 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2713.9 | uss_MB: 1466.8 | child_rss_MB: 22789.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:41,119] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:24:41,119] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:21,819] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:25:21,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:46,407] INFO : Step 20000/500000 rank0 | Train Loss: 4.4590 | Train Loss Unweighted: 2.6500 | Train Loss Components: Classification: 0.0126 | Classification_binary: 2.3815 | Regression_cos_phi: 0.0086 | Regression_energy: 0.0858 | Regression_eta: 0.0669 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1559 | Classification_binary: 2.3815 | Regression_cos_phi: 0.0769 | Regression_energy: 0.5310 | Regression_eta: 0.4073 | Regression_pt: 0.8242 | Regression_sin_phi: 0.0821 | LR: 4.07e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8030s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0033 | max_backward: 0.0268 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.2222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2714.0 | uss_MB: 1466.8 | child_rss_MB: 22789.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:25:46,409] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:25:46,459] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:25:46,459] INFO : Running validation on rank0 +rank0 [2026-11-14 04:25:46,460] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:25:46,460] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:25:46,540] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:25:46,613] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:25:46,683] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:25:46,732] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:25:46,779] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:25:46,824] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:25:46,870] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:25:46,915] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:25:46,988] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:25:47,063] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:25:47,137] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:25:47,185] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:25:47,231] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:25:47,276] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:25:47,323] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:25:47,368] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:25:47,438] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:25:47,442] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:25:47,443] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:25:47,445] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:25:47,446] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:25:47,448] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:25:47,449] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:25:47,451] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:25:47,452] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:25:47,454] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:25:47,454] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:25:47,456] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:25:47,457] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:25:47,459] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:25:47,460] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:25:47,462] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:25:48,415] INFO : VALIDATION | Step=20000/500000 | Train Loss=4.4590 | Valid Loss=6.5046 | Train Loss Unweighted=2.6500 | Valid Loss Unweighted=2.8795 | Stale=0 | ETA=589.4m +rank0 [2026-11-14 04:25:48,416] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:25:48,517] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:25:48,523] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:48,571] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:25:48,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:25:49,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:25:49,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:25:49,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:25:49,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:25:49,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:25:49,944] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:25:50,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:25:50,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:25:50,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:25:50,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:25:50,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:25:50,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:25:51,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:25:51,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:25:51,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:25:51,390] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:25:51,397] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:51,763] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:52,040] INFO : Plotted number of elements +rank0 [2026-11-14 04:25:52,871] INFO : Plotted jets +rank0 [2026-11-14 04:25:53,547] INFO : Plotted jet ratio +rank0 [2026-11-14 04:26:10,841] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:26:10,841] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:51,734] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:26:51,734] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:07,528] INFO : Step 21000/500000 rank0 | Train Loss: 4.2320 | Train Loss Unweighted: 1.6307 | Train Loss Components: Classification: 0.0067 | Classification_binary: 1.2646 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1487 | Regression_eta: 0.0140 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0176 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 1.2646 | Regression_cos_phi: 0.0951 | Regression_energy: 0.9202 | Regression_eta: 0.0853 | Regression_pt: 1.6229 | Regression_sin_phi: 0.1610 | LR: 4.28e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7696s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0918 | max_tensorboard_logging: 0.2266 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2742.9 | uss_MB: 1495.8 | child_rss_MB: 22790.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:27:33,572] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:27:33,573] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:14,849] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:28:14,849] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:21,590] INFO : Step 22000/500000 rank0 | Train Loss: 2.9613 | Train Loss Unweighted: 2.1714 | Train Loss Components: Classification: 0.0080 | Classification_binary: 2.0573 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0526 | Regression_eta: 0.0085 | Regression_pt: 0.0344 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0987 | Classification_binary: 2.0573 | Regression_cos_phi: 0.0462 | Regression_energy: 0.3253 | Regression_eta: 0.0514 | Regression_pt: 0.3319 | Regression_sin_phi: 0.0504 | LR: 4.50e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7620s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0266 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2191 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2744.1 | uss_MB: 1496.9 | child_rss_MB: 22788.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:28:21,592] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:28:21,643] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:57,158] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:28:57,159] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:35,994] INFO : Step 23000/500000 rank0 | Train Loss: 4.8482 | Train Loss Unweighted: 2.1989 | Train Loss Components: Classification: 0.0089 | Classification_binary: 1.7791 | Regression_cos_phi: 0.0374 | Regression_energy: 0.1214 | Regression_eta: 0.1535 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0238 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1101 | Classification_binary: 1.7791 | Regression_cos_phi: 0.3356 | Regression_energy: 0.7509 | Regression_eta: 0.9339 | Regression_pt: 0.7216 | Regression_sin_phi: 0.2170 | LR: 4.73e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7607s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.0858 | max_tensorboard_logging: 0.2162 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2747.0 | uss_MB: 1499.8 | child_rss_MB: 22790.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:29:39,056] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:29:39,056] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:19,988] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:30:19,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:49,573] INFO : Step 24000/500000 rank0 | Train Loss: 3.7633 | Train Loss Unweighted: 1.7395 | Train Loss Components: Classification: 0.0099 | Classification_binary: 1.4513 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1256 | Regression_eta: 0.0139 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1227 | Classification_binary: 1.4513 | Regression_cos_phi: 0.0901 | Regression_energy: 0.7772 | Regression_eta: 0.0848 | Regression_pt: 1.1912 | Regression_sin_phi: 0.0460 | LR: 4.97e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2579 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2747.0 | uss_MB: 1499.8 | child_rss_MB: 22787.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:30:49,574] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:30:49,621] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:31:01,755] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:31:01,755] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:42,984] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:31:42,984] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:03,904] INFO : Step 25000/500000 rank0 | Train Loss: 4.1874 | Train Loss Unweighted: 2.6505 | Train Loss Components: Classification: 0.0045 | Classification_binary: 2.4273 | Regression_cos_phi: 0.0128 | Regression_energy: 0.0879 | Regression_eta: 0.0241 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 2.4273 | Regression_cos_phi: 0.1150 | Regression_energy: 0.5440 | Regression_eta: 0.1467 | Regression_pt: 0.7899 | Regression_sin_phi: 0.1087 | LR: 5.22e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7567s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0024 | max_backward: 0.0265 | max_optimizer_step: 0.0847 | max_tensorboard_logging: 0.2166 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.0 | uss_MB: 1495.8 | child_rss_MB: 22786.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:32:25,146] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:32:25,147] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:06,218] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:33:06,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:17,745] INFO : Step 26000/500000 rank0 | Train Loss: 4.4076 | Train Loss Unweighted: 2.7216 | Train Loss Components: Classification: 0.0045 | Classification_binary: 2.4826 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1056 | Regression_eta: 0.0060 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 2.4826 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6531 | Regression_eta: 0.0368 | Regression_pt: 1.1083 | Regression_sin_phi: 0.0226 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7695s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2178 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.5 | uss_MB: 1496.3 | child_rss_MB: 22791.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:33:17,747] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:33:17,794] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:33:48,212] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:33:48,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:29,285] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:34:29,285] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:31,767] INFO : Step 27000/500000 rank0 | Train Loss: 3.4516 | Train Loss Unweighted: 1.4531 | Train Loss Components: Classification: 0.0095 | Classification_binary: 1.1713 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1244 | Regression_eta: 0.0071 | Regression_pt: 0.1297 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1169 | Classification_binary: 1.1713 | Regression_cos_phi: 0.0496 | Regression_energy: 0.7694 | Regression_eta: 0.0435 | Regression_pt: 1.2503 | Regression_sin_phi: 0.0506 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7974s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0034 | max_backward: 0.0270 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2168 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2746.7 | uss_MB: 1499.6 | child_rss_MB: 22790.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:11,264] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:35:11,264] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:45,605] INFO : Step 28000/500000 rank0 | Train Loss: 1.6957 | Train Loss Unweighted: 0.8891 | Train Loss Components: Classification: 0.0061 | Classification_binary: 0.7764 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0476 | Regression_eta: 0.0041 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.7764 | Regression_cos_phi: 0.0384 | Regression_energy: 0.2947 | Regression_eta: 0.0250 | Regression_pt: 0.4582 | Regression_sin_phi: 0.0283 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7560s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0267 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2165 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2747.2 | uss_MB: 1500.0 | child_rss_MB: 22790.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:45,607] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:35:45,653] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:35:52,980] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:35:52,981] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:34,076] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:36:34,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:59,563] INFO : Step 29000/500000 rank0 | Train Loss: 2.4380 | Train Loss Unweighted: 0.7145 | Train Loss Components: Classification: 0.0067 | Classification_binary: 0.4666 | Regression_cos_phi: 0.0021 | Regression_energy: 0.1206 | Regression_eta: 0.0046 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.4666 | Regression_cos_phi: 0.0188 | Regression_energy: 0.7460 | Regression_eta: 0.0281 | Regression_pt: 1.0703 | Regression_sin_phi: 0.0260 | LR: 6.29e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7833s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0268 | max_optimizer_step: 0.0865 | max_tensorboard_logging: 0.2216 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2747.2 | uss_MB: 1500.0 | child_rss_MB: 22790.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:37:16,033] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:37:16,033] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:56,605] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:37:56,605] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:38:13,019] INFO : Step 30000/500000 rank0 | Train Loss: 1.7242 | Train Loss Unweighted: 1.1886 | Train Loss Components: Classification: 0.0034 | Classification_binary: 1.1132 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0274 | Regression_eta: 0.0071 | Regression_pt: 0.0288 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 1.1132 | Regression_cos_phi: 0.0449 | Regression_energy: 0.1694 | Regression_eta: 0.0433 | Regression_pt: 0.2775 | Regression_sin_phi: 0.0342 | LR: 6.58e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7442s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0260 | max_loss: 0.0025 | max_backward: 0.0268 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.2107 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2743.5 | uss_MB: 1496.4 | child_rss_MB: 22790.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:38:13,021] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:38:13,069] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:38:13,069] INFO : Running validation on rank0 +rank0 [2026-11-14 04:38:13,070] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:38:13,070] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:38:13,150] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:38:13,222] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:38:13,291] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:38:13,341] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:38:13,387] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:38:13,432] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:38:13,479] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:38:13,527] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:38:13,601] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:38:13,676] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:38:13,752] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:38:13,800] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:38:13,844] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:38:13,889] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:38:13,936] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:38:13,981] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:38:14,057] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:38:14,061] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:38:14,063] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:38:14,065] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:38:14,066] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:38:14,067] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:38:14,069] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:38:14,071] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:38:14,072] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:38:14,073] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:38:14,074] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:38:14,075] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:38:14,077] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:38:14,077] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:38:14,079] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:38:14,080] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:38:14,981] INFO : VALIDATION | Step=30000/500000 | Train Loss=1.7242 | Valid Loss=7.4558 | Train Loss Unweighted=1.1886 | Valid Loss Unweighted=3.3457 | Stale=1 | ETA=579.7m +rank0 [2026-11-14 04:38:14,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:38:15,082] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:38:15,088] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:38:15,146] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:38:15,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:38:15,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:38:15,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:38:16,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:38:16,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:38:16,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:38:16,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:38:16,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:38:16,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:38:16,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:38:17,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:38:17,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:38:17,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:38:17,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:38:17,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:38:17,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:38:17,987] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:38:17,994] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:38:18,349] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:38:18,633] INFO : Plotted number of elements +rank0 [2026-11-14 04:38:19,456] INFO : Plotted jets +rank0 [2026-11-14 04:38:20,136] INFO : Plotted jet ratio +rank0 [2026-11-14 04:38:46,036] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:38:46,036] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:26,895] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:39:26,896] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:34,172] INFO : Step 31000/500000 rank0 | Train Loss: 2.8504 | Train Loss Unweighted: 1.3722 | Train Loss Components: Classification: 0.0043 | Classification_binary: 1.1585 | Regression_cos_phi: 0.0156 | Regression_energy: 0.0895 | Regression_eta: 0.0155 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 1.1585 | Regression_cos_phi: 0.1403 | Regression_energy: 0.5539 | Regression_eta: 0.0943 | Regression_pt: 0.7642 | Regression_sin_phi: 0.0864 | LR: 6.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8037s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0258 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2147 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2857.3 | uss_MB: 1610.2 | child_rss_MB: 22788.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:40:08,011] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:40:08,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:47,365] INFO : Step 32000/500000 rank0 | Train Loss: 4.3809 | Train Loss Unweighted: 2.6023 | Train Loss Components: Classification: 0.0123 | Classification_binary: 2.3515 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1076 | Regression_eta: 0.0121 | Regression_pt: 0.1075 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1521 | Classification_binary: 2.3515 | Regression_cos_phi: 0.0460 | Regression_energy: 0.6660 | Regression_eta: 0.0736 | Regression_pt: 1.0359 | Regression_sin_phi: 0.0558 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7574s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0024 | max_backward: 0.0266 | max_optimizer_step: 0.0825 | max_tensorboard_logging: 0.2101 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2857.3 | uss_MB: 1610.2 | child_rss_MB: 22790.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:40:47,367] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:40:47,414] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:40:49,883] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:40:49,883] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:31,200] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:41:31,201] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:01,409] INFO : Step 33000/500000 rank0 | Train Loss: 2.1991 | Train Loss Unweighted: 1.6123 | Train Loss Components: Classification: 0.0027 | Classification_binary: 1.5291 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0355 | Regression_eta: 0.0041 | Regression_pt: 0.0363 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 1.5291 | Regression_cos_phi: 0.0211 | Regression_energy: 0.2195 | Regression_eta: 0.0250 | Regression_pt: 0.3495 | Regression_sin_phi: 0.0211 | LR: 7.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7662s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2191 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2768.9 | uss_MB: 1521.7 | child_rss_MB: 22787.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:42:13,024] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 04:42:13,025] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:53,800] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 04:42:53,800] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:14,357] INFO : Step 34000/500000 rank0 | Train Loss: 2.8371 | Train Loss Unweighted: 1.5594 | Train Loss Components: Classification: 0.0059 | Classification_binary: 1.3792 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0786 | Regression_eta: 0.0057 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 1.3792 | Regression_cos_phi: 0.0384 | Regression_energy: 0.4860 | Regression_eta: 0.0345 | Regression_pt: 0.8055 | Regression_sin_phi: 0.0209 | LR: 7.83e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7445s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0848 | max_tensorboard_logging: 0.2043 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2769.8 | uss_MB: 1522.6 | child_rss_MB: 22791.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:43:14,358] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 04:43:14,405] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:43:34,874] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 04:43:34,874] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:15,534] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 04:44:15,534] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:27,783] INFO : Step 35000/500000 rank0 | Train Loss: 1.7831 | Train Loss Unweighted: 0.8870 | Train Loss Components: Classification: 0.0082 | Classification_binary: 0.7662 | Regression_cos_phi: 0.0141 | Regression_energy: 0.0325 | Regression_eta: 0.0100 | Regression_pt: 0.0323 | Regression_sin_phi: 0.0237 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1010 | Classification_binary: 0.7662 | Regression_cos_phi: 0.1267 | Regression_energy: 0.2011 | Regression_eta: 0.0606 | Regression_pt: 0.3110 | Regression_sin_phi: 0.2165 | LR: 8.16e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7536s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0854 | max_tensorboard_logging: 0.2103 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2772.4 | uss_MB: 1525.3 | child_rss_MB: 22791.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:44:57,484] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 04:44:57,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:38,525] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 04:45:38,526] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:41,581] INFO : Step 36000/500000 rank0 | Train Loss: 1.4189 | Train Loss Unweighted: 0.5654 | Train Loss Components: Classification: 0.0012 | Classification_binary: 0.4431 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0553 | Regression_eta: 0.0034 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0153 | Classification_binary: 0.4431 | Regression_cos_phi: 0.0283 | Regression_energy: 0.3420 | Regression_eta: 0.0205 | Regression_pt: 0.5561 | Regression_sin_phi: 0.0135 | LR: 8.50e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7530s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0261 | max_loss: 0.0024 | max_backward: 0.0266 | max_optimizer_step: 0.0845 | max_tensorboard_logging: 0.2101 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2772.4 | uss_MB: 1525.3 | child_rss_MB: 22790.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:45:41,583] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 04:45:41,629] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:46:19,800] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 04:46:19,800] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:53,826] INFO : Step 37000/500000 rank0 | Train Loss: 1.8079 | Train Loss Unweighted: 1.3006 | Train Loss Components: Classification: 0.0045 | Classification_binary: 1.2293 | Regression_cos_phi: 0.0015 | Regression_energy: 0.0306 | Regression_eta: 0.0035 | Regression_pt: 0.0283 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 1.2293 | Regression_cos_phi: 0.0132 | Regression_energy: 0.1892 | Regression_eta: 0.0210 | Regression_pt: 0.2730 | Regression_sin_phi: 0.0271 | LR: 8.85e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7659s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0025 | max_backward: 0.0255 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2772.4 | uss_MB: 1525.3 | child_rss_MB: 22790.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:47:00,337] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 04:47:00,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:40,502] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 04:47:40,502] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:06,086] INFO : Step 38000/500000 rank0 | Train Loss: 2.1077 | Train Loss Unweighted: 1.2613 | Train Loss Components: Classification: 0.0034 | Classification_binary: 1.1426 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0472 | Regression_eta: 0.0050 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 1.1426 | Regression_cos_phi: 0.0659 | Regression_energy: 0.2919 | Regression_eta: 0.0306 | Regression_pt: 0.4955 | Regression_sin_phi: 0.0393 | LR: 9.21e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7950s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.2189 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2769.5 | uss_MB: 1522.3 | child_rss_MB: 22791.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:48:06,088] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 04:48:06,137] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:48:21,789] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 04:48:21,790] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:02,924] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 04:49:02,925] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:19,538] INFO : Step 39000/500000 rank0 | Train Loss: 1.5429 | Train Loss Unweighted: 0.7889 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.6796 | Regression_cos_phi: 0.0108 | Regression_energy: 0.0349 | Regression_eta: 0.0170 | Regression_pt: 0.0330 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0175 | Classification_binary: 0.6796 | Regression_cos_phi: 0.0967 | Regression_energy: 0.2159 | Regression_eta: 0.1037 | Regression_pt: 0.3184 | Regression_sin_phi: 0.1110 | LR: 9.57e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7735s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0025 | max_backward: 0.0262 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2198 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2772.5 | uss_MB: 1525.4 | child_rss_MB: 22788.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:49:44,383] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 04:49:44,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:25,790] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 04:50:25,791] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:33,709] INFO : Step 40000/500000 rank0 | Train Loss: 0.7901 | Train Loss Unweighted: 0.2619 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0273 | Regression_eta: 0.0078 | Regression_pt: 0.0279 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0073 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0535 | Regression_energy: 0.1689 | Regression_eta: 0.0477 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0575 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7659s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0266 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2235 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2772.8 | uss_MB: 1525.6 | child_rss_MB: 22787.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:50:33,711] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 04:50:33,760] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 04:50:33,760] INFO : Running validation on rank0 +rank0 [2026-11-14 04:50:33,760] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:50:33,760] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:50:33,840] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 04:50:33,914] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:50:33,983] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:50:34,033] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:50:34,079] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:50:34,124] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:50:34,170] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:50:34,216] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:50:34,292] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:50:34,369] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:50:34,444] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:50:34,491] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:50:34,537] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:50:34,583] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:50:34,629] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:50:34,674] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:50:34,744] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:50:34,748] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:50:34,750] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:50:34,751] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:50:34,753] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:50:34,754] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:50:34,756] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:50:34,757] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:50:34,759] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:50:34,760] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:50:34,761] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:50:34,762] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:50:34,763] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:50:34,764] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:50:34,765] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:50:34,767] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:50:35,705] INFO : VALIDATION | Step=40000/500000 | Train Loss=0.7901 | Valid Loss=8.9350 | Train Loss Unweighted=0.2619 | Valid Loss Unweighted=4.8757 | Stale=2 | ETA=567.5m +rank0 [2026-11-14 04:50:35,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:50:35,806] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:50:35,811] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:50:35,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:50:36,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:50:36,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:50:36,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:50:36,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:50:36,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:50:37,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:50:37,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:50:37,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:50:37,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:50:37,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:50:37,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:50:38,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:50:38,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:50:38,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:50:38,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:50:38,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:50:38,681] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:50:38,688] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:50:39,043] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:50:39,315] INFO : Plotted number of elements +rank0 [2026-11-14 04:50:40,158] INFO : Plotted jets +rank0 [2026-11-14 04:50:40,847] INFO : Plotted jet ratio +rank0 [2026-11-14 04:51:15,000] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 04:51:15,000] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:54,610] INFO : Step 41000/500000 rank0 | Train Loss: 2.4202 | Train Loss Unweighted: 1.2545 | Train Loss Components: Classification: 0.0011 | Classification_binary: 1.0866 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0750 | Regression_eta: 0.0066 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0133 | Classification_binary: 1.0866 | Regression_cos_phi: 0.0486 | Regression_energy: 0.4640 | Regression_eta: 0.0401 | Regression_pt: 0.7416 | Regression_sin_phi: 0.0261 | LR: 1.03e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7953s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0305 | max_loss: 0.0024 | max_backward: 0.0257 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2229 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2793.5 | uss_MB: 1546.4 | child_rss_MB: 22790.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:51:56,500] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 04:51:56,500] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:37,246] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 04:52:37,247] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:07,750] INFO : Step 42000/500000 rank0 | Train Loss: 2.6728 | Train Loss Unweighted: 0.9313 | Train Loss Components: Classification: 0.0051 | Classification_binary: 0.7008 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0354 | Regression_eta: 0.0168 | Regression_pt: 0.0351 | Regression_sin_phi: 0.0665 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.7008 | Regression_cos_phi: 0.6428 | Regression_energy: 0.2192 | Regression_eta: 0.1022 | Regression_pt: 0.3383 | Regression_sin_phi: 0.6066 | LR: 1.07e-04 | DataLoad Time: 0.0092s | Model Forward Time: 0.7678s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0269 | max_optimizer_step: 0.0872 | max_tensorboard_logging: 0.2136 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.1 | uss_MB: 1547.0 | child_rss_MB: 22792.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:53:07,752] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 04:53:07,801] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 04:53:18,788] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 04:53:18,788] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:00,493] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 04:54:00,497] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:21,916] INFO : Step 43000/500000 rank0 | Train Loss: 0.8374 | Train Loss Unweighted: 0.1798 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.0884 | Regression_cos_phi: 0.0154 | Regression_energy: 0.0283 | Regression_eta: 0.0048 | Regression_pt: 0.0281 | Regression_sin_phi: 0.0147 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0025 | Classification_binary: 0.0884 | Regression_cos_phi: 0.1380 | Regression_energy: 0.1748 | Regression_eta: 0.0292 | Regression_pt: 0.2707 | Regression_sin_phi: 0.1338 | LR: 1.11e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7979s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0265 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.2192 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2797.7 | uss_MB: 1550.6 | child_rss_MB: 22791.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:54:42,020] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 04:54:42,020] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:55:23,305] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 04:55:23,305] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:55:35,969] INFO : Step 44000/500000 rank0 | Train Loss: 2.7090 | Train Loss Unweighted: 1.2181 | Train Loss Components: Classification: 0.0040 | Classification_binary: 1.0023 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1042 | Regression_eta: 0.0058 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 1.0023 | Regression_cos_phi: 0.0457 | Regression_energy: 0.6446 | Regression_eta: 0.0353 | Regression_pt: 0.9129 | Regression_sin_phi: 0.0189 | LR: 1.15e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7688s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2200 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2798.6 | uss_MB: 1551.5 | child_rss_MB: 22791.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:55:35,972] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 04:55:36,019] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 04:56:04,883] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 04:56:04,883] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:46,464] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 04:56:46,464] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:50,071] INFO : Step 45000/500000 rank0 | Train Loss: 0.4125 | Train Loss Unweighted: 0.0956 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.0492 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0196 | Regression_eta: 0.0045 | Regression_pt: 0.0172 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0097 | Classification_binary: 0.0492 | Regression_cos_phi: 0.0203 | Regression_energy: 0.1213 | Regression_eta: 0.0273 | Regression_pt: 0.1656 | Regression_sin_phi: 0.0191 | LR: 1.19e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7631s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0256 | max_optimizer_step: 0.0847 | max_tensorboard_logging: 0.2141 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2798.6 | uss_MB: 1551.5 | child_rss_MB: 22791.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:57:27,870] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 04:57:27,870] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:03,690] INFO : Step 46000/500000 rank0 | Train Loss: 1.0537 | Train Loss Unweighted: 0.6161 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.5522 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0275 | Regression_eta: 0.0048 | Regression_pt: 0.0254 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0034 | Classification_binary: 0.5522 | Regression_cos_phi: 0.0271 | Regression_energy: 0.1700 | Regression_eta: 0.0292 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0266 | LR: 1.23e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7982s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2459 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.1 | uss_MB: 1547.0 | child_rss_MB: 22790.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:58:03,692] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 04:58:03,743] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 04:58:09,906] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 04:58:09,906] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:51,153] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 04:58:51,153] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:17,338] INFO : Step 47000/500000 rank0 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4885 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.4204 | Regression_cos_phi: 0.0017 | Regression_energy: 0.0287 | Regression_eta: 0.0056 | Regression_pt: 0.0298 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0014 | Classification_binary: 0.4204 | Regression_cos_phi: 0.0157 | Regression_energy: 0.1777 | Regression_eta: 0.0341 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0192 | LR: 1.27e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.7708s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0025 | max_backward: 0.0255 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2158 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2797.8 | uss_MB: 1550.7 | child_rss_MB: 22790.7 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:59:32,131] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 04:59:32,131] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:13,298] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 05:00:13,299] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:30,881] INFO : Step 48000/500000 rank0 | Train Loss: 1.2665 | Train Loss Unweighted: 0.6059 | Train Loss Components: Classification: 0.0013 | Classification_binary: 0.5096 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0420 | Regression_eta: 0.0062 | Regression_pt: 0.0353 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0160 | Classification_binary: 0.5096 | Regression_cos_phi: 0.0665 | Regression_energy: 0.2597 | Regression_eta: 0.0380 | Regression_pt: 0.3402 | Regression_sin_phi: 0.0366 | LR: 1.31e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7652s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0267 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2148 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2797.8 | uss_MB: 1550.7 | child_rss_MB: 22790.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:00:30,883] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 05:00:30,931] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:00:55,295] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:00:55,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:36,383] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:01:36,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:44,862] INFO : Step 49000/500000 rank0 | Train Loss: 0.6601 | Train Loss Unweighted: 0.2125 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0237 | Regression_eta: 0.0043 | Regression_pt: 0.0275 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0173 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0208 | Regression_energy: 0.1465 | Regression_eta: 0.0262 | Regression_pt: 0.2654 | Regression_sin_phi: 0.0343 | LR: 1.36e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7636s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0269 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2142 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2797.9 | uss_MB: 1550.8 | child_rss_MB: 22789.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:02:18,275] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:02:18,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:57,877] INFO : Step 50000/500000 rank0 | Train Loss: 1.8363 | Train Loss Unweighted: 0.7330 | Train Loss Components: Classification: 0.0017 | Classification_binary: 0.5738 | Regression_cos_phi: 0.0066 | Regression_energy: 0.0724 | Regression_eta: 0.0058 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0208 | Classification_binary: 0.5738 | Regression_cos_phi: 0.0595 | Regression_energy: 0.4482 | Regression_eta: 0.0352 | Regression_pt: 0.6674 | Regression_sin_phi: 0.0313 | LR: 1.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7499s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0255 | max_optimizer_step: 0.0836 | max_tensorboard_logging: 0.2144 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2793.7 | uss_MB: 1611.0 | child_rss_MB: 22792.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:02:57,878] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:02:57,928] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:02:57,929] INFO : Running validation on rank0 +rank0 [2026-11-14 05:02:57,929] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:02:57,929] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:02:58,014] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:02:58,087] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:02:58,157] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:02:58,206] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:02:58,252] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:02:58,300] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:02:58,347] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:02:58,395] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:02:58,459] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:02:58,535] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:02:58,611] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:02:58,658] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:02:58,703] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:02:58,749] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:02:58,796] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:02:58,843] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:02:58,916] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:02:58,920] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:02:58,922] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:02:58,923] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:02:58,925] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:02:58,926] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:02:58,928] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:02:58,929] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:02:58,931] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:02:58,932] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:02:58,933] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:02:58,934] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:02:58,935] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:02:58,937] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:02:58,938] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:02:58,940] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:02:59,848] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.8363 | Valid Loss=11.6288 | Train Loss Unweighted=0.7330 | Valid Loss Unweighted=6.7884 | Stale=3 | ETA=555.7m +rank0 [2026-11-14 05:02:59,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:02:59,947] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:02:59,952] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:03:00,000] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:03:00,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:03:00,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:03:00,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:03:00,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:03:01,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:03:01,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:03:01,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:03:01,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:03:01,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:03:01,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:03:01,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:03:02,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:03:02,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:03:02,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:03:02,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:03:02,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:03:02,833] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 05:03:02,841] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:03:03,198] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:03:03,477] INFO : Plotted number of elements +rank0 [2026-11-14 05:03:04,347] INFO : Plotted jets +rank0 [2026-11-14 05:03:05,043] INFO : Plotted jet ratio +rank0 [2026-11-14 05:03:06,152] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:03:06,152] INFO : Training completed. Total time on device 0: 61.853min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..caf5809061bcd908bd277ad3e39926cc4b923ccf --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/train.log.0 @@ -0,0 +1,997 @@ +rank0 [2026-11-14 04:01:12,417] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 04:01:12,417] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 04:01:12,417] INFO : Instantiating model +rank0 [2026-11-14 04:01:12,435] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 04:01:12,435] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 04:01:12,437] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 04:01:12,437] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 04:01:12,437] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 04:01:12,437] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 04:01:12,437] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,439] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 04:01:12,439] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,441] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 04:01:12,441] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,444] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 04:01:12,444] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,446] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 04:01:12,446] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,448] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 04:01:12,448] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,451] INFO : Backbone initialization took 0.01s +rank0 [2026-11-14 04:01:12,451] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 04:01:12,451] INFO : Initializing output DNNs +rank0 [2026-11-14 04:01:12,452] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,454] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:01:12,460] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 04:01:12,460] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 04:01:12,460] INFO : jagged_attention enabled=True +rank0 [2026-11-14 04:01:12,460] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 04:01:12,460] INFO : task_queries enabled=True +rank0 [2026-11-14 04:01:12,460] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 04:01:12,460] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 04:01:12,461] INFO : classification_query parameters: 256 +rank0 [2026-11-14 04:01:12,461] INFO : regression_query parameters: 256 +rank0 [2026-11-14 04:01:12,461] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 04:01:12,461] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 04:01:12,461] INFO : output_mode=elementwise +rank0 [2026-11-14 04:01:12,461] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 04:01:12,461] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 04:01:12,461] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 04:01:12,461] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 04:01:12,461] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 04:01:12,461] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 04:01:12,461] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 04:01:12,461] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 04:01:12,462] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 04:01:12,462] INFO : MLPF __init__ done +rank0 [2026-11-14 04:01:12,462] INFO : Instantiated model +rank0 [2026-11-14 04:01:12,463] INFO : Moving model to device rank=0 +rank0 [2026-11-14 04:01:12,468] INFO : Moved model to device rank=0 +rank0 [2026-11-14 04:01:12,469] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 04:01:12,537] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 04:01:12,537] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 04:01:12,538] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 04:01:12,538] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 04:01:12,538] INFO : Total parameters: 3910930 +rank0 [2026-11-14 04:01:12,540] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 04:01:12,541] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296 +rank0 [2026-11-14 04:01:12,541] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296 +rank0 [2026-11-14 04:01:12,541] INFO : Getting dataloaders +rank0 [2026-11-14 04:01:12,541] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:01:14,177] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,262] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,328] INFO : train_dataset: cld_edm_z_tautau_pf, 898 +rank0 [2026-11-14 04:01:14,364] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,414] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,459] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,502] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,540] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,586] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:01:14,621] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:01:14,621] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 04:01:14,622] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 04:01:14,622] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 04:01:14,622] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:01:14,631] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,642] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,651] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,661] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,671] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,681] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,691] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,701] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,710] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,720] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:01:14,720] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 04:01:14,721] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 04:01:14,754] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:01:14,955] INFO : Got dataloaders +rank0 [2026-11-14 04:01:14,955] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 04:01:14,955] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 04:01:14,955] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 04:01:14,959] INFO : Creating train_iterator +rank0 [2026-11-14 04:01:14,959] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:01:14,992] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:01:15,206] INFO : Created train_iterator +rank0 [2026-11-14 04:01:23,354] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.4305033683776855, 'Classification': 0.60180958122015, 'Regression_pt': 0.7710086131095886, 'Regression_eta': 1.221010594367981, 'Regression_sin_phi': 0.8146398048102855, 'Regression_cos_phi': 0.8275578145682811, 'Regression_energy': 1.2010143318772315}, weights={'Classification_binary': 1.0, 'Classification': 12.34693431854248, 'Regression_pt': 9.637380599975586, 'Regression_eta': 6.085535526275635, 'Regression_sin_phi': 9.12121295928955, 'Regression_cos_phi': 8.978833198547363, 'Regression_energy': 6.186856746673584} +rank0 [2026-11-14 04:01:56,610] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 04:01:56,611] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:02:28,538] INFO : Step 1000/500000 rank0 | Train Loss: 11.6954 | Train Loss Unweighted: 5.3087 | Train Loss Components: Classification: 0.0862 | Classification_binary: 4.4440 | Regression_cos_phi: 0.0229 | Regression_energy: 0.3268 | Regression_eta: 0.0453 | Regression_pt: 0.3618 | Regression_sin_phi: 0.0217 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 1.0637 | Classification_binary: 4.4440 | Regression_cos_phi: 0.2056 | Regression_energy: 2.0218 | Regression_eta: 0.2759 | Regression_pt: 3.4868 | Regression_sin_phi: 0.1975 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7577s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0023 | max_backward: 0.0253 | max_optimizer_step: 0.0841 | max_tensorboard_logging: 0.2243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2615.4 | uss_MB: 1360.9 | child_rss_MB: 22799.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:02:37,277] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:02:37,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:17,518] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:03:17,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:03:40,918] INFO : Step 2000/500000 rank0 | Train Loss: 8.3653 | Train Loss Unweighted: 4.0104 | Train Loss Components: Classification: 0.0523 | Classification_binary: 3.4109 | Regression_cos_phi: 0.0136 | Regression_energy: 0.2462 | Regression_eta: 0.0281 | Regression_pt: 0.2464 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.6461 | Classification_binary: 3.4109 | Regression_cos_phi: 0.1218 | Regression_energy: 1.5230 | Regression_eta: 0.1712 | Regression_pt: 2.3748 | Regression_sin_phi: 0.1175 | LR: 2.02e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7477s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0256 | max_loss: 0.0023 | max_backward: 0.0250 | max_optimizer_step: 0.0821 | max_tensorboard_logging: 0.2202 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2616.1 | uss_MB: 1361.8 | child_rss_MB: 22796.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:03:40,920] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:03:58,438] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:03:58,439] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:39,552] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:04:39,552] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:04:53,829] INFO : Step 3000/500000 rank0 | Train Loss: 9.0523 | Train Loss Unweighted: 3.9949 | Train Loss Components: Classification: 0.0984 | Classification_binary: 3.3296 | Regression_cos_phi: 0.0134 | Regression_energy: 0.2506 | Regression_eta: 0.0206 | Regression_pt: 0.2667 | Regression_sin_phi: 0.0156 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 1.2143 | Classification_binary: 3.3296 | Regression_cos_phi: 0.1203 | Regression_energy: 1.5504 | Regression_eta: 0.1254 | Regression_pt: 2.5702 | Regression_sin_phi: 0.1420 | LR: 2.05e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7479s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0832 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2620.1 | uss_MB: 1366.1 | child_rss_MB: 22789.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:05:20,761] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:05:20,761] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:01,446] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:06:01,446] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:06:06,921] INFO : Step 4000/500000 rank0 | Train Loss: 8.2691 | Train Loss Unweighted: 3.6304 | Train Loss Components: Classification: 0.0350 | Classification_binary: 2.9813 | Regression_cos_phi: 0.0210 | Regression_energy: 0.2652 | Regression_eta: 0.0335 | Regression_pt: 0.2658 | Regression_sin_phi: 0.0286 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.4324 | Classification_binary: 2.9813 | Regression_cos_phi: 0.1882 | Regression_energy: 1.6410 | Regression_eta: 0.2039 | Regression_pt: 2.5614 | Regression_sin_phi: 0.2608 | LR: 2.08e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7581s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0024 | max_backward: 0.0262 | max_optimizer_step: 0.0836 | max_tensorboard_logging: 0.2275 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2620.5 | uss_MB: 1366.5 | child_rss_MB: 22790.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:06:06,924] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:06:43,188] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:06:43,188] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:07:20,910] INFO : Step 5000/500000 rank0 | Train Loss: 8.7609 | Train Loss Unweighted: 4.3335 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.7084 | Regression_cos_phi: 0.0116 | Regression_energy: 0.2736 | Regression_eta: 0.0281 | Regression_pt: 0.2697 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.3882 | Classification_binary: 3.7084 | Regression_cos_phi: 0.1038 | Regression_energy: 1.6927 | Regression_eta: 0.1713 | Regression_pt: 2.5991 | Regression_sin_phi: 0.0975 | LR: 2.13e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7439s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0822 | max_tensorboard_logging: 0.2202 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2618.0 | uss_MB: 1364.0 | child_rss_MB: 22788.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:07:25,150] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:07:25,150] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:05,572] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:08:05,573] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:08:33,792] INFO : Step 6000/500000 rank0 | Train Loss: 6.7430 | Train Loss Unweighted: 3.1303 | Train Loss Components: Classification: 0.0108 | Classification_binary: 2.6204 | Regression_cos_phi: 0.0125 | Regression_energy: 0.2235 | Regression_eta: 0.0105 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1332 | Classification_binary: 2.6204 | Regression_cos_phi: 0.1123 | Regression_energy: 1.3827 | Regression_eta: 0.0639 | Regression_pt: 2.3604 | Regression_sin_phi: 0.0701 | LR: 2.19e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7434s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0023 | max_backward: 0.0252 | max_optimizer_step: 0.0827 | max_tensorboard_logging: 0.2177 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2618.0 | uss_MB: 1364.0 | child_rss_MB: 22787.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:08:33,793] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:08:46,764] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:08:46,764] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:27,195] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:09:27,195] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:09:46,457] INFO : Step 7000/500000 rank0 | Train Loss: 11.5347 | Train Loss Unweighted: 4.6921 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.6982 | Regression_cos_phi: 0.0222 | Regression_energy: 0.3933 | Regression_eta: 0.1194 | Regression_pt: 0.4085 | Regression_sin_phi: 0.0257 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.3060 | Classification_binary: 3.6982 | Regression_cos_phi: 0.1991 | Regression_energy: 2.4334 | Regression_eta: 0.7268 | Regression_pt: 3.9366 | Regression_sin_phi: 0.2346 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7422s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0023 | max_backward: 0.0262 | max_optimizer_step: 0.0835 | max_tensorboard_logging: 0.2134 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2619.0 | uss_MB: 1364.9 | child_rss_MB: 22785.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:10:09,144] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:10:09,144] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:10:49,997] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:10:49,997] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:11:00,242] INFO : Step 8000/500000 rank0 | Train Loss: 4.5125 | Train Loss Unweighted: 2.6202 | Train Loss Components: Classification: 0.0070 | Classification_binary: 2.3511 | Regression_cos_phi: 0.0033 | Regression_energy: 0.1220 | Regression_eta: 0.0072 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0861 | Classification_binary: 2.3511 | Regression_cos_phi: 0.0298 | Regression_energy: 0.7548 | Regression_eta: 0.0438 | Regression_pt: 1.2082 | Regression_sin_phi: 0.0387 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7434s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0263 | max_optimizer_step: 0.0826 | max_tensorboard_logging: 0.2162 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2619.0 | uss_MB: 1365.0 | child_rss_MB: 22788.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:11:00,243] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:11:00,306] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:11:31,478] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:11:31,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:12:12,345] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:12:12,346] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:12:13,594] INFO : Step 9000/500000 rank0 | Train Loss: 6.3280 | Train Loss Unweighted: 3.4132 | Train Loss Components: Classification: 0.0056 | Classification_binary: 2.9941 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1939 | Regression_eta: 0.0113 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 2.9941 | Regression_cos_phi: 0.0955 | Regression_energy: 1.1997 | Regression_eta: 0.0686 | Regression_pt: 1.8272 | Regression_sin_phi: 0.0741 | LR: 2.43e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7482s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0023 | max_backward: 0.0253 | max_optimizer_step: 0.0828 | max_tensorboard_logging: 0.2210 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2617.0 | uss_MB: 1363.0 | child_rss_MB: 22789.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 04:12:53,664] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:12:53,664] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:13:26,042] INFO : Step 10000/500000 rank0 | Train Loss: 7.6845 | Train Loss Unweighted: 3.6372 | Train Loss Components: Classification: 0.0349 | Classification_binary: 3.0725 | Regression_cos_phi: 0.0117 | Regression_energy: 0.2522 | Regression_eta: 0.0107 | Regression_pt: 0.2368 | Regression_sin_phi: 0.0184 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.4315 | Classification_binary: 3.0725 | Regression_cos_phi: 0.1054 | Regression_energy: 1.5601 | Regression_eta: 0.0649 | Regression_pt: 2.2819 | Regression_sin_phi: 0.1681 | LR: 2.52e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7458s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0251 | max_optimizer_step: 0.0818 | max_tensorboard_logging: 0.2200 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2617.6 | uss_MB: 1363.6 | child_rss_MB: 22789.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:13:26,044] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:13:26,104] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:13:26,105] INFO : Running validation on rank0 +rank0 [2026-11-14 04:13:26,105] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:13:26,105] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:13:26,205] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:13:26,308] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:13:26,357] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:13:26,396] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:13:26,435] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:13:26,468] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:13:26,507] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:13:26,547] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:13:26,583] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:13:26,639] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:13:26,698] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:13:26,735] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:13:26,774] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:13:26,806] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:13:26,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:13:26,884] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:13:26,917] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:13:26,919] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:13:26,921] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:13:26,922] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:13:26,924] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:13:26,926] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:13:26,927] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:13:26,929] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:13:26,930] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:13:26,932] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:13:26,932] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:13:26,934] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:13:26,935] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:13:26,937] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:13:26,938] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:13:26,940] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:13:27,885] INFO : VALIDATION | Step=10000/500000 | Train Loss=7.6845 | Valid Loss=6.5424 | Train Loss Unweighted=3.6372 | Valid Loss Unweighted=2.8993 | Stale=0 | ETA=598.6m +rank0 [2026-11-14 04:13:27,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:13:27,986] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:13:27,991] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:28,038] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:13:28,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:13:28,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:13:28,789] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:13:28,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:13:29,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:13:29,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:13:29,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:13:29,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:13:29,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:13:29,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:13:30,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:13:30,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:13:30,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:13:30,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:13:30,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:13:30,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:13:30,822] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:13:30,829] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:31,179] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:13:31,520] INFO : Plotted number of elements +rank0 [2026-11-14 04:13:32,398] INFO : Plotted jets +rank0 [2026-11-14 04:13:33,099] INFO : Plotted jet ratio +rank0 [2026-11-14 04:13:41,656] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:13:41,656] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:21,592] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:14:21,592] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:14:45,242] INFO : Step 11000/500000 rank0 | Train Loss: 3.9926 | Train Loss Unweighted: 2.3634 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.1422 | Regression_cos_phi: 0.0083 | Regression_energy: 0.0880 | Regression_eta: 0.0088 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.2885 | Classification_binary: 2.1422 | Regression_cos_phi: 0.0745 | Regression_energy: 0.5445 | Regression_eta: 0.0538 | Regression_pt: 0.8200 | Regression_sin_phi: 0.0691 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8005s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.2238 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2767.4 | uss_MB: 1520.2 | child_rss_MB: 22790.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:15:02,040] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:15:02,040] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:42,730] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:15:42,730] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:15:57,883] INFO : Step 12000/500000 rank0 | Train Loss: 3.4987 | Train Loss Unweighted: 2.0849 | Train Loss Components: Classification: 0.0052 | Classification_binary: 1.8857 | Regression_cos_phi: 0.0070 | Regression_energy: 0.0491 | Regression_eta: 0.0392 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0149 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 1.8857 | Regression_cos_phi: 0.0631 | Regression_energy: 0.3038 | Regression_eta: 0.2385 | Regression_pt: 0.8085 | Regression_sin_phi: 0.1355 | LR: 2.75e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7635s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0254 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.2205 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2767.4 | uss_MB: 1520.2 | child_rss_MB: 22790.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:15:57,885] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:15:57,934] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:16:23,923] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:16:23,923] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:17:03,939] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:17:03,939] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:17:09,870] INFO : Step 13000/500000 rank0 | Train Loss: 2.7313 | Train Loss Unweighted: 1.5699 | Train Loss Components: Classification: 0.0060 | Classification_binary: 1.4045 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0723 | Regression_eta: 0.0082 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 1.4045 | Regression_cos_phi: 0.0287 | Regression_energy: 0.4471 | Regression_eta: 0.0501 | Regression_pt: 0.6913 | Regression_sin_phi: 0.0359 | LR: 2.88e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7621s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2183 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2710.5 | uss_MB: 1463.3 | child_rss_MB: 22788.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:17:45,249] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 04:17:45,250] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:18:22,683] INFO : Step 14000/500000 rank0 | Train Loss: 6.5522 | Train Loss Unweighted: 2.5690 | Train Loss Components: Classification: 0.0074 | Classification_binary: 2.0008 | Regression_cos_phi: 0.0150 | Regression_energy: 0.2651 | Regression_eta: 0.0048 | Regression_pt: 0.2700 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0913 | Classification_binary: 2.0008 | Regression_cos_phi: 0.1344 | Regression_energy: 1.6402 | Regression_eta: 0.0292 | Regression_pt: 2.6023 | Regression_sin_phi: 0.0540 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7620s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0843 | max_tensorboard_logging: 0.2239 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2711.2 | uss_MB: 1464.0 | child_rss_MB: 22787.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:18:22,685] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:18:22,733] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:18:26,358] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 04:18:26,358] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:07,561] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 04:19:07,561] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:19:36,892] INFO : Step 15000/500000 rank0 | Train Loss: 7.2494 | Train Loss Unweighted: 3.8746 | Train Loss Components: Classification: 0.0139 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0069 | Regression_energy: 0.2298 | Regression_eta: 0.0076 | Regression_pt: 0.2127 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1716 | Classification_binary: 3.3920 | Regression_cos_phi: 0.0618 | Regression_energy: 1.4217 | Regression_eta: 0.0465 | Regression_pt: 2.0498 | Regression_sin_phi: 0.1060 | LR: 3.17e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7591s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0265 | max_optimizer_step: 0.0847 | max_tensorboard_logging: 0.2173 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2713.9 | uss_MB: 1466.7 | child_rss_MB: 22786.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:19:49,634] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 04:19:49,635] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:30,773] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 04:20:30,773] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:20:50,615] INFO : Step 16000/500000 rank0 | Train Loss: 4.3721 | Train Loss Unweighted: 2.4909 | Train Loss Components: Classification: 0.0272 | Classification_binary: 2.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.1089 | Regression_eta: 0.0021 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.3363 | Classification_binary: 2.2369 | Regression_cos_phi: 0.0182 | Regression_energy: 0.6738 | Regression_eta: 0.0127 | Regression_pt: 1.0595 | Regression_sin_phi: 0.0347 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7555s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0872 | max_tensorboard_logging: 0.2196 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2713.9 | uss_MB: 1466.7 | child_rss_MB: 22786.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:20:50,617] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:20:50,666] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:21:12,451] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 04:21:12,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:21:53,512] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 04:21:53,513] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:22:04,466] INFO : Step 17000/500000 rank0 | Train Loss: 3.2942 | Train Loss Unweighted: 2.2404 | Train Loss Components: Classification: 0.0123 | Classification_binary: 2.0962 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0597 | Regression_eta: 0.0044 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1516 | Classification_binary: 2.0962 | Regression_cos_phi: 0.0244 | Regression_energy: 0.3693 | Regression_eta: 0.0267 | Regression_pt: 0.5950 | Regression_sin_phi: 0.0310 | LR: 3.51e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7504s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2089 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.4 | uss_MB: 1463.2 | child_rss_MB: 22786.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:22:35,854] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 04:22:35,854] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:16,976] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 04:23:16,976] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:23:18,851] INFO : Step 18000/500000 rank0 | Train Loss: 2.2629 | Train Loss Unweighted: 1.4820 | Train Loss Components: Classification: 0.0098 | Classification_binary: 1.3755 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0409 | Regression_eta: 0.0056 | Regression_pt: 0.0422 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1210 | Classification_binary: 1.3755 | Regression_cos_phi: 0.0240 | Regression_energy: 0.2529 | Regression_eta: 0.0340 | Regression_pt: 0.4069 | Regression_sin_phi: 0.0487 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2116 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2711.0 | uss_MB: 1463.8 | child_rss_MB: 22791.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:23:18,852] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:23:18,900] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:23:59,092] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 04:23:59,092] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:24:33,190] INFO : Step 19000/500000 rank0 | Train Loss: 4.2935 | Train Loss Unweighted: 2.1450 | Train Loss Components: Classification: 0.0341 | Classification_binary: 1.8557 | Regression_cos_phi: 0.0042 | Regression_energy: 0.1192 | Regression_eta: 0.0072 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.4213 | Classification_binary: 1.8557 | Regression_cos_phi: 0.0373 | Regression_energy: 0.7376 | Regression_eta: 0.0439 | Regression_pt: 1.1498 | Regression_sin_phi: 0.0478 | LR: 3.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7815s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0026 | max_backward: 0.0269 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2713.9 | uss_MB: 1466.8 | child_rss_MB: 22789.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:24:41,119] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 04:24:41,119] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:21,819] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 04:25:21,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:25:46,407] INFO : Step 20000/500000 rank0 | Train Loss: 4.4590 | Train Loss Unweighted: 2.6500 | Train Loss Components: Classification: 0.0126 | Classification_binary: 2.3815 | Regression_cos_phi: 0.0086 | Regression_energy: 0.0858 | Regression_eta: 0.0669 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1559 | Classification_binary: 2.3815 | Regression_cos_phi: 0.0769 | Regression_energy: 0.5310 | Regression_eta: 0.4073 | Regression_pt: 0.8242 | Regression_sin_phi: 0.0821 | LR: 4.07e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8030s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0033 | max_backward: 0.0268 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.2222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2714.0 | uss_MB: 1466.8 | child_rss_MB: 22789.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:25:46,409] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:25:46,459] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 04:25:46,459] INFO : Running validation on rank0 +rank0 [2026-11-14 04:25:46,460] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:25:46,460] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:25:46,540] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 04:25:46,613] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:25:46,683] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:25:46,732] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:25:46,779] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:25:46,824] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:25:46,870] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:25:46,915] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:25:46,988] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:25:47,063] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:25:47,137] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:25:47,185] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:25:47,231] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:25:47,276] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:25:47,323] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:25:47,368] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:25:47,438] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:25:47,442] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:25:47,443] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:25:47,445] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:25:47,446] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:25:47,448] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:25:47,449] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:25:47,451] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:25:47,452] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:25:47,454] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:25:47,454] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:25:47,456] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:25:47,457] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:25:47,459] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:25:47,460] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:25:47,462] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:25:48,415] INFO : VALIDATION | Step=20000/500000 | Train Loss=4.4590 | Valid Loss=6.5046 | Train Loss Unweighted=2.6500 | Valid Loss Unweighted=2.8795 | Stale=0 | ETA=589.4m +rank0 [2026-11-14 04:25:48,416] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:25:48,517] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:25:48,523] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:48,571] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:25:48,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:25:49,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:25:49,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:25:49,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:25:49,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:25:49,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:25:49,944] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:25:50,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:25:50,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:25:50,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:25:50,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:25:50,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:25:50,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:25:51,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:25:51,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:25:51,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:25:51,390] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:25:51,397] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:51,763] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:25:52,040] INFO : Plotted number of elements +rank0 [2026-11-14 04:25:52,871] INFO : Plotted jets +rank0 [2026-11-14 04:25:53,547] INFO : Plotted jet ratio +rank0 [2026-11-14 04:26:10,841] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 04:26:10,841] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:26:51,734] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 04:26:51,734] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:27:07,528] INFO : Step 21000/500000 rank0 | Train Loss: 4.2320 | Train Loss Unweighted: 1.6307 | Train Loss Components: Classification: 0.0067 | Classification_binary: 1.2646 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1487 | Regression_eta: 0.0140 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0176 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 1.2646 | Regression_cos_phi: 0.0951 | Regression_energy: 0.9202 | Regression_eta: 0.0853 | Regression_pt: 1.6229 | Regression_sin_phi: 0.1610 | LR: 4.28e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7696s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0918 | max_tensorboard_logging: 0.2266 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2742.9 | uss_MB: 1495.8 | child_rss_MB: 22790.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:27:33,572] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 04:27:33,573] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:14,849] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 04:28:14,849] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:28:21,590] INFO : Step 22000/500000 rank0 | Train Loss: 2.9613 | Train Loss Unweighted: 2.1714 | Train Loss Components: Classification: 0.0080 | Classification_binary: 2.0573 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0526 | Regression_eta: 0.0085 | Regression_pt: 0.0344 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0987 | Classification_binary: 2.0573 | Regression_cos_phi: 0.0462 | Regression_energy: 0.3253 | Regression_eta: 0.0514 | Regression_pt: 0.3319 | Regression_sin_phi: 0.0504 | LR: 4.50e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7620s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0266 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2191 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2744.1 | uss_MB: 1496.9 | child_rss_MB: 22788.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:28:21,592] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:28:21,643] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 04:28:57,158] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 04:28:57,159] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:29:35,994] INFO : Step 23000/500000 rank0 | Train Loss: 4.8482 | Train Loss Unweighted: 2.1989 | Train Loss Components: Classification: 0.0089 | Classification_binary: 1.7791 | Regression_cos_phi: 0.0374 | Regression_energy: 0.1214 | Regression_eta: 0.1535 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0238 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1101 | Classification_binary: 1.7791 | Regression_cos_phi: 0.3356 | Regression_energy: 0.7509 | Regression_eta: 0.9339 | Regression_pt: 0.7216 | Regression_sin_phi: 0.2170 | LR: 4.73e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7607s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.0858 | max_tensorboard_logging: 0.2162 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2747.0 | uss_MB: 1499.8 | child_rss_MB: 22790.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:29:39,056] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 04:29:39,056] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:19,988] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 04:30:19,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:30:49,573] INFO : Step 24000/500000 rank0 | Train Loss: 3.7633 | Train Loss Unweighted: 1.7395 | Train Loss Components: Classification: 0.0099 | Classification_binary: 1.4513 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1256 | Regression_eta: 0.0139 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1227 | Classification_binary: 1.4513 | Regression_cos_phi: 0.0901 | Regression_energy: 0.7772 | Regression_eta: 0.0848 | Regression_pt: 1.1912 | Regression_sin_phi: 0.0460 | LR: 4.97e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2579 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2747.0 | uss_MB: 1499.8 | child_rss_MB: 22787.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:30:49,574] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:30:49,621] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 04:31:01,755] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 04:31:01,755] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:31:42,984] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 04:31:42,984] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:32:03,904] INFO : Step 25000/500000 rank0 | Train Loss: 4.1874 | Train Loss Unweighted: 2.6505 | Train Loss Components: Classification: 0.0045 | Classification_binary: 2.4273 | Regression_cos_phi: 0.0128 | Regression_energy: 0.0879 | Regression_eta: 0.0241 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 2.4273 | Regression_cos_phi: 0.1150 | Regression_energy: 0.5440 | Regression_eta: 0.1467 | Regression_pt: 0.7899 | Regression_sin_phi: 0.1087 | LR: 5.22e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7567s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0024 | max_backward: 0.0265 | max_optimizer_step: 0.0847 | max_tensorboard_logging: 0.2166 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.0 | uss_MB: 1495.8 | child_rss_MB: 22786.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:32:25,146] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 04:32:25,147] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:06,218] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 04:33:06,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:33:17,745] INFO : Step 26000/500000 rank0 | Train Loss: 4.4076 | Train Loss Unweighted: 2.7216 | Train Loss Components: Classification: 0.0045 | Classification_binary: 2.4826 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1056 | Regression_eta: 0.0060 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 2.4826 | Regression_cos_phi: 0.0487 | Regression_energy: 0.6531 | Regression_eta: 0.0368 | Regression_pt: 1.1083 | Regression_sin_phi: 0.0226 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7695s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2178 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.5 | uss_MB: 1496.3 | child_rss_MB: 22791.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:33:17,747] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:33:17,794] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 04:33:48,212] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 04:33:48,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:29,285] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 04:34:29,285] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:34:31,767] INFO : Step 27000/500000 rank0 | Train Loss: 3.4516 | Train Loss Unweighted: 1.4531 | Train Loss Components: Classification: 0.0095 | Classification_binary: 1.1713 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1244 | Regression_eta: 0.0071 | Regression_pt: 0.1297 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1169 | Classification_binary: 1.1713 | Regression_cos_phi: 0.0496 | Regression_energy: 0.7694 | Regression_eta: 0.0435 | Regression_pt: 1.2503 | Regression_sin_phi: 0.0506 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7974s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0034 | max_backward: 0.0270 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2168 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2746.7 | uss_MB: 1499.6 | child_rss_MB: 22790.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:11,264] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 04:35:11,264] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:35:45,605] INFO : Step 28000/500000 rank0 | Train Loss: 1.6957 | Train Loss Unweighted: 0.8891 | Train Loss Components: Classification: 0.0061 | Classification_binary: 0.7764 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0476 | Regression_eta: 0.0041 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.7764 | Regression_cos_phi: 0.0384 | Regression_energy: 0.2947 | Regression_eta: 0.0250 | Regression_pt: 0.4582 | Regression_sin_phi: 0.0283 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7560s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0267 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2165 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2747.2 | uss_MB: 1500.0 | child_rss_MB: 22790.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:35:45,607] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:35:45,653] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 04:35:52,980] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 04:35:52,981] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:34,076] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 04:36:34,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:59,563] INFO : Step 29000/500000 rank0 | Train Loss: 2.4380 | Train Loss Unweighted: 0.7145 | Train Loss Components: Classification: 0.0067 | Classification_binary: 0.4666 | Regression_cos_phi: 0.0021 | Regression_energy: 0.1206 | Regression_eta: 0.0046 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.4666 | Regression_cos_phi: 0.0188 | Regression_energy: 0.7460 | Regression_eta: 0.0281 | Regression_pt: 1.0703 | Regression_sin_phi: 0.0260 | LR: 6.29e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7833s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0268 | max_optimizer_step: 0.0865 | max_tensorboard_logging: 0.2216 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2747.2 | uss_MB: 1500.0 | child_rss_MB: 22790.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:37:16,033] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 04:37:16,033] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:37:56,605] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 04:37:56,605] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:38:13,019] INFO : Step 30000/500000 rank0 | Train Loss: 1.7242 | Train Loss Unweighted: 1.1886 | Train Loss Components: Classification: 0.0034 | Classification_binary: 1.1132 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0274 | Regression_eta: 0.0071 | Regression_pt: 0.0288 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 1.1132 | Regression_cos_phi: 0.0449 | Regression_energy: 0.1694 | Regression_eta: 0.0433 | Regression_pt: 0.2775 | Regression_sin_phi: 0.0342 | LR: 6.58e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7442s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0260 | max_loss: 0.0025 | max_backward: 0.0268 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.2107 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2743.5 | uss_MB: 1496.4 | child_rss_MB: 22790.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:38:13,021] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:38:13,069] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 04:38:13,069] INFO : Running validation on rank0 +rank0 [2026-11-14 04:38:13,070] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:38:13,070] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:38:13,150] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 04:38:13,222] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:38:13,291] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:38:13,341] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:38:13,387] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:38:13,432] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:38:13,479] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:38:13,527] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:38:13,601] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:38:13,676] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:38:13,752] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:38:13,800] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:38:13,844] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:38:13,889] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:38:13,936] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:38:13,981] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:38:14,057] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:38:14,061] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:38:14,063] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:38:14,065] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:38:14,066] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:38:14,067] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:38:14,069] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:38:14,071] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:38:14,072] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:38:14,073] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:38:14,074] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:38:14,075] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:38:14,077] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:38:14,077] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:38:14,079] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:38:14,080] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:38:14,981] INFO : VALIDATION | Step=30000/500000 | Train Loss=1.7242 | Valid Loss=7.4558 | Train Loss Unweighted=1.1886 | Valid Loss Unweighted=3.3457 | Stale=1 | ETA=579.7m +rank0 [2026-11-14 04:38:14,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:38:15,082] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:38:15,088] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:38:15,146] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:38:15,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:38:15,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:38:15,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:38:16,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:38:16,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:38:16,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:38:16,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:38:16,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:38:16,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:38:16,995] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:38:17,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:38:17,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:38:17,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:38:17,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:38:17,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:38:17,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:38:17,987] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:38:17,994] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:38:18,349] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:38:18,633] INFO : Plotted number of elements +rank0 [2026-11-14 04:38:19,456] INFO : Plotted jets +rank0 [2026-11-14 04:38:20,136] INFO : Plotted jet ratio +rank0 [2026-11-14 04:38:46,036] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 04:38:46,036] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:26,895] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 04:39:26,896] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:34,172] INFO : Step 31000/500000 rank0 | Train Loss: 2.8504 | Train Loss Unweighted: 1.3722 | Train Loss Components: Classification: 0.0043 | Classification_binary: 1.1585 | Regression_cos_phi: 0.0156 | Regression_energy: 0.0895 | Regression_eta: 0.0155 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 1.1585 | Regression_cos_phi: 0.1403 | Regression_energy: 0.5539 | Regression_eta: 0.0943 | Regression_pt: 0.7642 | Regression_sin_phi: 0.0864 | LR: 6.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8037s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0258 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2147 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2857.3 | uss_MB: 1610.2 | child_rss_MB: 22788.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:40:08,011] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 04:40:08,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:47,365] INFO : Step 32000/500000 rank0 | Train Loss: 4.3809 | Train Loss Unweighted: 2.6023 | Train Loss Components: Classification: 0.0123 | Classification_binary: 2.3515 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1076 | Regression_eta: 0.0121 | Regression_pt: 0.1075 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1521 | Classification_binary: 2.3515 | Regression_cos_phi: 0.0460 | Regression_energy: 0.6660 | Regression_eta: 0.0736 | Regression_pt: 1.0359 | Regression_sin_phi: 0.0558 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7574s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0024 | max_backward: 0.0266 | max_optimizer_step: 0.0825 | max_tensorboard_logging: 0.2101 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2857.3 | uss_MB: 1610.2 | child_rss_MB: 22790.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:40:47,367] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:40:47,414] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 04:40:49,883] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 04:40:49,883] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:31,200] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 04:41:31,201] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:01,409] INFO : Step 33000/500000 rank0 | Train Loss: 2.1991 | Train Loss Unweighted: 1.6123 | Train Loss Components: Classification: 0.0027 | Classification_binary: 1.5291 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0355 | Regression_eta: 0.0041 | Regression_pt: 0.0363 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 1.5291 | Regression_cos_phi: 0.0211 | Regression_energy: 0.2195 | Regression_eta: 0.0250 | Regression_pt: 0.3495 | Regression_sin_phi: 0.0211 | LR: 7.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7662s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2191 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2768.9 | uss_MB: 1521.7 | child_rss_MB: 22787.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:42:13,024] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 04:42:13,025] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:42:53,800] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 04:42:53,800] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:14,357] INFO : Step 34000/500000 rank0 | Train Loss: 2.8371 | Train Loss Unweighted: 1.5594 | Train Loss Components: Classification: 0.0059 | Classification_binary: 1.3792 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0786 | Regression_eta: 0.0057 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 1.3792 | Regression_cos_phi: 0.0384 | Regression_energy: 0.4860 | Regression_eta: 0.0345 | Regression_pt: 0.8055 | Regression_sin_phi: 0.0209 | LR: 7.83e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7445s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0848 | max_tensorboard_logging: 0.2043 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2769.8 | uss_MB: 1522.6 | child_rss_MB: 22791.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:43:14,358] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 04:43:14,405] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 04:43:34,874] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 04:43:34,874] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:15,534] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 04:44:15,534] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:44:27,783] INFO : Step 35000/500000 rank0 | Train Loss: 1.7831 | Train Loss Unweighted: 0.8870 | Train Loss Components: Classification: 0.0082 | Classification_binary: 0.7662 | Regression_cos_phi: 0.0141 | Regression_energy: 0.0325 | Regression_eta: 0.0100 | Regression_pt: 0.0323 | Regression_sin_phi: 0.0237 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.1010 | Classification_binary: 0.7662 | Regression_cos_phi: 0.1267 | Regression_energy: 0.2011 | Regression_eta: 0.0606 | Regression_pt: 0.3110 | Regression_sin_phi: 0.2165 | LR: 8.16e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7536s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0854 | max_tensorboard_logging: 0.2103 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2772.4 | uss_MB: 1525.3 | child_rss_MB: 22791.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:44:57,484] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 04:44:57,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:38,525] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 04:45:38,526] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:41,581] INFO : Step 36000/500000 rank0 | Train Loss: 1.4189 | Train Loss Unweighted: 0.5654 | Train Loss Components: Classification: 0.0012 | Classification_binary: 0.4431 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0553 | Regression_eta: 0.0034 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0153 | Classification_binary: 0.4431 | Regression_cos_phi: 0.0283 | Regression_energy: 0.3420 | Regression_eta: 0.0205 | Regression_pt: 0.5561 | Regression_sin_phi: 0.0135 | LR: 8.50e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7530s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0261 | max_loss: 0.0024 | max_backward: 0.0266 | max_optimizer_step: 0.0845 | max_tensorboard_logging: 0.2101 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2772.4 | uss_MB: 1525.3 | child_rss_MB: 22790.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:45:41,583] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 04:45:41,629] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 04:46:19,800] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 04:46:19,800] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:53,826] INFO : Step 37000/500000 rank0 | Train Loss: 1.8079 | Train Loss Unweighted: 1.3006 | Train Loss Components: Classification: 0.0045 | Classification_binary: 1.2293 | Regression_cos_phi: 0.0015 | Regression_energy: 0.0306 | Regression_eta: 0.0035 | Regression_pt: 0.0283 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 1.2293 | Regression_cos_phi: 0.0132 | Regression_energy: 0.1892 | Regression_eta: 0.0210 | Regression_pt: 0.2730 | Regression_sin_phi: 0.0271 | LR: 8.85e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7659s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0025 | max_backward: 0.0255 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2772.4 | uss_MB: 1525.3 | child_rss_MB: 22790.4 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:47:00,337] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 04:47:00,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:40,502] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 04:47:40,502] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:06,086] INFO : Step 38000/500000 rank0 | Train Loss: 2.1077 | Train Loss Unweighted: 1.2613 | Train Loss Components: Classification: 0.0034 | Classification_binary: 1.1426 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0472 | Regression_eta: 0.0050 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 1.1426 | Regression_cos_phi: 0.0659 | Regression_energy: 0.2919 | Regression_eta: 0.0306 | Regression_pt: 0.4955 | Regression_sin_phi: 0.0393 | LR: 9.21e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7950s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.2189 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2769.5 | uss_MB: 1522.3 | child_rss_MB: 22791.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:48:06,088] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 04:48:06,137] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 04:48:21,789] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 04:48:21,790] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:02,924] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 04:49:02,925] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:49:19,538] INFO : Step 39000/500000 rank0 | Train Loss: 1.5429 | Train Loss Unweighted: 0.7889 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.6796 | Regression_cos_phi: 0.0108 | Regression_energy: 0.0349 | Regression_eta: 0.0170 | Regression_pt: 0.0330 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0175 | Classification_binary: 0.6796 | Regression_cos_phi: 0.0967 | Regression_energy: 0.2159 | Regression_eta: 0.1037 | Regression_pt: 0.3184 | Regression_sin_phi: 0.1110 | LR: 9.57e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7735s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0025 | max_backward: 0.0262 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2198 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2772.5 | uss_MB: 1525.4 | child_rss_MB: 22788.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:49:44,383] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 04:49:44,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:25,790] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 04:50:25,791] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:33,709] INFO : Step 40000/500000 rank0 | Train Loss: 0.7901 | Train Loss Unweighted: 0.2619 | Train Loss Components: Classification: 0.0006 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0273 | Regression_eta: 0.0078 | Regression_pt: 0.0279 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0073 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0535 | Regression_energy: 0.1689 | Regression_eta: 0.0477 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0575 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7659s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0266 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2235 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2772.8 | uss_MB: 1525.6 | child_rss_MB: 22787.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:50:33,711] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 04:50:33,760] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 04:50:33,760] INFO : Running validation on rank0 +rank0 [2026-11-14 04:50:33,760] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:50:33,760] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:50:33,840] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 04:50:33,914] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:50:33,983] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:50:34,033] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:50:34,079] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:50:34,124] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:50:34,170] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:50:34,216] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:50:34,292] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:50:34,369] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:50:34,444] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:50:34,491] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:50:34,537] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:50:34,583] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:50:34,629] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:50:34,674] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:50:34,744] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:50:34,748] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:50:34,750] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:50:34,751] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:50:34,753] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:50:34,754] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:50:34,756] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:50:34,757] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:50:34,759] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:50:34,760] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:50:34,761] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:50:34,762] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:50:34,763] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:50:34,764] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:50:34,765] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:50:34,767] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:50:35,705] INFO : VALIDATION | Step=40000/500000 | Train Loss=0.7901 | Valid Loss=8.9350 | Train Loss Unweighted=0.2619 | Valid Loss Unweighted=4.8757 | Stale=2 | ETA=567.5m +rank0 [2026-11-14 04:50:35,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:50:35,806] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:50:35,811] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:50:35,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:50:36,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:50:36,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:50:36,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:50:36,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:50:36,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:50:37,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:50:37,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:50:37,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:50:37,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:50:37,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:50:37,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:50:38,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:50:38,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:50:38,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:50:38,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:50:38,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:50:38,681] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 04:50:38,688] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:50:39,043] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:50:39,315] INFO : Plotted number of elements +rank0 [2026-11-14 04:50:40,158] INFO : Plotted jets +rank0 [2026-11-14 04:50:40,847] INFO : Plotted jet ratio +rank0 [2026-11-14 04:51:15,000] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 04:51:15,000] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:51:54,610] INFO : Step 41000/500000 rank0 | Train Loss: 2.4202 | Train Loss Unweighted: 1.2545 | Train Loss Components: Classification: 0.0011 | Classification_binary: 1.0866 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0750 | Regression_eta: 0.0066 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0133 | Classification_binary: 1.0866 | Regression_cos_phi: 0.0486 | Regression_energy: 0.4640 | Regression_eta: 0.0401 | Regression_pt: 0.7416 | Regression_sin_phi: 0.0261 | LR: 1.03e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7953s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0305 | max_loss: 0.0024 | max_backward: 0.0257 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2229 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2793.5 | uss_MB: 1546.4 | child_rss_MB: 22790.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:51:56,500] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 04:51:56,500] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:37,246] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 04:52:37,247] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:53:07,750] INFO : Step 42000/500000 rank0 | Train Loss: 2.6728 | Train Loss Unweighted: 0.9313 | Train Loss Components: Classification: 0.0051 | Classification_binary: 0.7008 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0354 | Regression_eta: 0.0168 | Regression_pt: 0.0351 | Regression_sin_phi: 0.0665 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.7008 | Regression_cos_phi: 0.6428 | Regression_energy: 0.2192 | Regression_eta: 0.1022 | Regression_pt: 0.3383 | Regression_sin_phi: 0.6066 | LR: 1.07e-04 | DataLoad Time: 0.0092s | Model Forward Time: 0.7678s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0269 | max_optimizer_step: 0.0872 | max_tensorboard_logging: 0.2136 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.1 | uss_MB: 1547.0 | child_rss_MB: 22792.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:53:07,752] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 04:53:07,801] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 04:53:18,788] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 04:53:18,788] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:00,493] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 04:54:00,497] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:21,916] INFO : Step 43000/500000 rank0 | Train Loss: 0.8374 | Train Loss Unweighted: 0.1798 | Train Loss Components: Classification: 0.0002 | Classification_binary: 0.0884 | Regression_cos_phi: 0.0154 | Regression_energy: 0.0283 | Regression_eta: 0.0048 | Regression_pt: 0.0281 | Regression_sin_phi: 0.0147 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0025 | Classification_binary: 0.0884 | Regression_cos_phi: 0.1380 | Regression_energy: 0.1748 | Regression_eta: 0.0292 | Regression_pt: 0.2707 | Regression_sin_phi: 0.1338 | LR: 1.11e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7979s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0265 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.2192 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2797.7 | uss_MB: 1550.6 | child_rss_MB: 22791.4 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:54:42,020] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 04:54:42,020] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:55:23,305] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 04:55:23,305] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:55:35,969] INFO : Step 44000/500000 rank0 | Train Loss: 2.7090 | Train Loss Unweighted: 1.2181 | Train Loss Components: Classification: 0.0040 | Classification_binary: 1.0023 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1042 | Regression_eta: 0.0058 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 1.0023 | Regression_cos_phi: 0.0457 | Regression_energy: 0.6446 | Regression_eta: 0.0353 | Regression_pt: 0.9129 | Regression_sin_phi: 0.0189 | LR: 1.15e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7688s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2200 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2798.6 | uss_MB: 1551.5 | child_rss_MB: 22791.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:55:35,972] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 04:55:36,019] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 04:56:04,883] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 04:56:04,883] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:46,464] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 04:56:46,464] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:50,071] INFO : Step 45000/500000 rank0 | Train Loss: 0.4125 | Train Loss Unweighted: 0.0956 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.0492 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0196 | Regression_eta: 0.0045 | Regression_pt: 0.0172 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0097 | Classification_binary: 0.0492 | Regression_cos_phi: 0.0203 | Regression_energy: 0.1213 | Regression_eta: 0.0273 | Regression_pt: 0.1656 | Regression_sin_phi: 0.0191 | LR: 1.19e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7631s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0256 | max_optimizer_step: 0.0847 | max_tensorboard_logging: 0.2141 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2798.6 | uss_MB: 1551.5 | child_rss_MB: 22791.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:57:27,870] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 04:57:27,870] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:03,690] INFO : Step 46000/500000 rank0 | Train Loss: 1.0537 | Train Loss Unweighted: 0.6161 | Train Loss Components: Classification: 0.0003 | Classification_binary: 0.5522 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0275 | Regression_eta: 0.0048 | Regression_pt: 0.0254 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0034 | Classification_binary: 0.5522 | Regression_cos_phi: 0.0271 | Regression_energy: 0.1700 | Regression_eta: 0.0292 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0266 | LR: 1.23e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7982s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2459 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.1 | uss_MB: 1547.0 | child_rss_MB: 22790.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:58:03,692] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 04:58:03,743] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 04:58:09,906] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 04:58:09,906] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:51,153] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 04:58:51,153] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:17,338] INFO : Step 47000/500000 rank0 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4885 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.4204 | Regression_cos_phi: 0.0017 | Regression_energy: 0.0287 | Regression_eta: 0.0056 | Regression_pt: 0.0298 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0014 | Classification_binary: 0.4204 | Regression_cos_phi: 0.0157 | Regression_energy: 0.1777 | Regression_eta: 0.0341 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0192 | LR: 1.27e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.7708s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0025 | max_backward: 0.0255 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2158 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2797.8 | uss_MB: 1550.7 | child_rss_MB: 22790.7 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:59:32,131] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 04:59:32,131] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:13,298] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 05:00:13,299] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:00:30,881] INFO : Step 48000/500000 rank0 | Train Loss: 1.2665 | Train Loss Unweighted: 0.6059 | Train Loss Components: Classification: 0.0013 | Classification_binary: 0.5096 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0420 | Regression_eta: 0.0062 | Regression_pt: 0.0353 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0160 | Classification_binary: 0.5096 | Regression_cos_phi: 0.0665 | Regression_energy: 0.2597 | Regression_eta: 0.0380 | Regression_pt: 0.3402 | Regression_sin_phi: 0.0366 | LR: 1.31e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7652s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0267 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2148 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2797.8 | uss_MB: 1550.7 | child_rss_MB: 22790.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:00:30,883] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 05:00:30,931] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:00:55,295] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 05:00:55,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:36,383] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 05:01:36,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:44,862] INFO : Step 49000/500000 rank0 | Train Loss: 0.6601 | Train Loss Unweighted: 0.2125 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0237 | Regression_eta: 0.0043 | Regression_pt: 0.0275 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0173 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0208 | Regression_energy: 0.1465 | Regression_eta: 0.0262 | Regression_pt: 0.2654 | Regression_sin_phi: 0.0343 | LR: 1.36e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7636s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0269 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2142 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2797.9 | uss_MB: 1550.8 | child_rss_MB: 22789.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:02:18,275] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 05:02:18,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:57,877] INFO : Step 50000/500000 rank0 | Train Loss: 1.8363 | Train Loss Unweighted: 0.7330 | Train Loss Components: Classification: 0.0017 | Classification_binary: 0.5738 | Regression_cos_phi: 0.0066 | Regression_energy: 0.0724 | Regression_eta: 0.0058 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.3469 | Classification_binary: 1.0000 | Regression_cos_phi: 8.9788 | Regression_energy: 6.1869 | Regression_eta: 6.0855 | Regression_pt: 9.6374 | Regression_sin_phi: 9.1212 | Task Weighted Losses: Classification: 0.0208 | Classification_binary: 0.5738 | Regression_cos_phi: 0.0595 | Regression_energy: 0.4482 | Regression_eta: 0.0352 | Regression_pt: 0.6674 | Regression_sin_phi: 0.0313 | LR: 1.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7499s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0255 | max_optimizer_step: 0.0836 | max_tensorboard_logging: 0.2144 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2793.7 | uss_MB: 1611.0 | child_rss_MB: 22792.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:02:57,878] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 05:02:57,928] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:02:57,929] INFO : Running validation on rank0 +rank0 [2026-11-14 05:02:57,929] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:02:57,929] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:02:58,014] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 05:02:58,087] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:02:58,157] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:02:58,206] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:02:58,252] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:02:58,300] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:02:58,347] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:02:58,395] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:02:58,459] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:02:58,535] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:02:58,611] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:02:58,658] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:02:58,703] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:02:58,749] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:02:58,796] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:02:58,843] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:02:58,916] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:02:58,920] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:02:58,922] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:02:58,923] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:02:58,925] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:02:58,926] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:02:58,928] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:02:58,929] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:02:58,931] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:02:58,932] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:02:58,933] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:02:58,934] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:02:58,935] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:02:58,937] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:02:58,938] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:02:58,940] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:02:59,848] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.8363 | Valid Loss=11.6288 | Train Loss Unweighted=0.7330 | Valid Loss Unweighted=6.7884 | Stale=3 | ETA=555.7m +rank0 [2026-11-14 05:02:59,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:02:59,947] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:02:59,952] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:03:00,000] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:03:00,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:03:00,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:03:00,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:03:00,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:03:01,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:03:01,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:03:01,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:03:01,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:03:01,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:03:01,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:03:01,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:03:02,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:03:02,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:03:02,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:03:02,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:03:02,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:03:02,833] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 05:03:02,841] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:03:03,198] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_4mm_seed12345_20261114_040046_949296/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:03:03,477] INFO : Plotted number of elements +rank0 [2026-11-14 05:03:04,347] INFO : Plotted jets +rank0 [2026-11-14 05:03:05,043] INFO : Plotted jet ratio +rank0 [2026-11-14 05:03:06,152] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 05:03:06,152] INFO : Training completed. Total time on device 0: 61.853min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..8cc25739b008379e8a3ab4cccab18a5fe3aee3ab --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fa234a185abf0f4696a686f9ecc71f0d9f327a0cb222fae280c299300948205 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-50000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..758e1e265b9f591bf65a977e2c7a572f3722302f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd9ddb66590ee58bcc652f91fb8c4722e553e17549114f73918234356b7de626 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..fbf6e5321b676a2ee688912e00cd608e72d3fe57 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 2.306260824203491, "Classification": 0.01241420116275549, "Regression_pt": 0.15578991174697876, "Regression_eta": 0.008025571703910828, "Regression_sin_phi": 0.004934059921652079, "Regression_cos_phi": 0.0050485869869589806, "Regression_energy": 0.1544686257839203, "Total": 5.039328575134277, "Total_unweighted": 2.6469417815096676}, "train_group": {}, "valid": {"Classification_binary": 2.44966983795166, "Classification": 0.017079783603549004, "Regression_pt": 0.18673117458820343, "Regression_eta": 0.036619413644075394, "Regression_sin_phi": 0.009733833372592926, "Regression_cos_phi": 0.010395020246505737, "Regression_energy": 0.1905478984117508, "Total": 6.035886764526367, "Total_unweighted": 2.9007769618183374, "metrics/particle/class_1/efficiency": 0.8432989716529846, "metrics/particle/class_1/pid_accuracy": 0.9731051325798035, "metrics/particle/class_2/efficiency": 0.09090909361839294, "metrics/particle/class_2/pid_accuracy": 0.5, "metrics/particle/class_3/efficiency": 0.6166982650756836, "metrics/particle/class_3/pid_accuracy": 0.9815384745597839, "metrics/particle/class_4/efficiency": 0.7916666865348816, "metrics/particle/class_4/pid_accuracy": 0.8947368264198303, "metrics/particle/class_5/efficiency": 0.9268292784690857, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": -0.2280000001192093, "metrics/particle/count/mae": 0.6520000100135803, "metrics/particle/count/prediction_mean": 4.619999885559082, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.12894143164157867, "metrics/particle/event/energy_response_mean": 1.0015331506729126, "metrics/particle/event/met_abs_error": 3.3560149669647217, "metrics/particle/event/scalar_pt_relative_abs_error": 0.1290036141872406, "metrics/particle/event/scalar_pt_response_mean": 1.0111639499664307, "metrics/particle/event/vector_pt_closure": 0.1348000466823578, "metrics/particle/matched/delta_eta_abs_mean": 0.028517240658402443, "metrics/particle/matched/delta_phi_abs_mean": 0.023888997733592987, "metrics/particle/matched/delta_r_mean": 0.04147008806467056, "metrics/particle/matched/energy_relative_abs_mean": 0.08545729517936707, "metrics/particle/matched/pid_accuracy": 0.9707207083702087, "metrics/particle/matched/pt_relative_abs_mean": 0.0840805321931839, "metrics/particle/matching/duplicate_fraction": 0.03030303120613098, "metrics/particle/matching/efficiency": 0.7326732873916626, "metrics/particle/matching/f1": 0.750316858291626, "metrics/particle/matching/purity": 0.768831193447113, "metrics/particle/matching/target_coverage_dr0p05": 0.5107260942459106, "metrics/particle/matching/target_coverage_dr0p10": 0.7656765580177307, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7153465151786804, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8201320171356201}, "step_train_time": 0.6439917087554932, "step_valid_time": 2.566815606027376, "step_total_time": 1071.7200376987457, "elapsed_training_loop_seconds": 1071.7200376987457, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.996660437459552, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08444058688235312, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 399, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 385, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9193548387096774, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9279069767441861, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9236111111111112, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07209302325581396, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8870967741935484, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8953488372093024, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8912037037037037, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9193548387096774, "step_test_time": 4.518141333013773, "step_plot_time": 3.4492143169627525, "step_validation_cycle_time": 11.015536257007625} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..6ca7f01588a045ec553830cdda006dcb8341b395 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 0.5976752638816833, "Classification": 0.00871189869940281, "Regression_pt": 0.036494698375463486, "Regression_eta": 0.003100208705291152, "Regression_sin_phi": 0.002155028283596039, "Regression_cos_phi": 0.0026956158690154552, "Regression_energy": 0.03483037278056145, "Total": 1.3325259685516357, "Total_unweighted": 0.6856630865950137}, "train_group": {}, "valid": {"Classification_binary": 2.31653094291687, "Classification": 0.01499423012137413, "Regression_pt": 0.15362457931041718, "Regression_eta": 0.03450829163193703, "Regression_sin_phi": 0.0083082290366292, "Regression_cos_phi": 0.008775821886956692, "Regression_energy": 0.15964077413082123, "Total": 5.329381942749023, "Total_unweighted": 2.6963828690350056, "metrics/particle/class_1/efficiency": 0.9010308980941772, "metrics/particle/class_1/pid_accuracy": 0.9816933870315552, "metrics/particle/class_2/efficiency": 0.3181818127632141, "metrics/particle/class_2/pid_accuracy": 0.5714285969734192, "metrics/particle/class_3/efficiency": 0.669829249382019, "metrics/particle/class_3/pid_accuracy": 0.9830028414726257, "metrics/particle/class_4/efficiency": 0.8645833134651184, "metrics/particle/class_4/pid_accuracy": 0.9036144614219666, "metrics/particle/class_5/efficiency": 0.9878048896789551, "metrics/particle/class_5/pid_accuracy": 0.9753086566925049, "metrics/particle/count/bias_mean": -0.06800000369548798, "metrics/particle/count/mae": 0.628000020980835, "metrics/particle/count/prediction_mean": 4.78000020980835, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.09753541648387909, "metrics/particle/event/energy_response_mean": 0.97920161485672, "metrics/particle/event/met_abs_error": 2.3755455017089844, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09303925186395645, "metrics/particle/event/scalar_pt_response_mean": 0.97859787940979, "metrics/particle/event/vector_pt_closure": 0.1008542850613594, "metrics/particle/matched/delta_eta_abs_mean": 0.025042982771992683, "metrics/particle/matched/delta_phi_abs_mean": 0.02179967239499092, "metrics/particle/matched/delta_r_mean": 0.037060901522636414, "metrics/particle/matched/energy_relative_abs_mean": 0.08641066402196884, "metrics/particle/matched/pid_accuracy": 0.9719042778015137, "metrics/particle/matched/pt_relative_abs_mean": 0.08542865514755249, "metrics/particle/matching/duplicate_fraction": 0.029288703575730324, "metrics/particle/matching/efficiency": 0.7929043173789978, "metrics/particle/matching/f1": 0.7985043525695801, "metrics/particle/matching/purity": 0.804184079170227, "metrics/particle/matching/target_coverage_dr0p05": 0.6006600856781006, "metrics/particle/matching/target_coverage_dr0p10": 0.8168317079544067, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7483498454093933, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8547855019569397}, "step_train_time": 0.5676760673522949, "step_valid_time": 2.8081017789663747, "step_total_time": 2149.922733068466, "elapsed_training_loop_seconds": 2149.922733068466, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9777114772626474, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0824277306420611, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 431, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 414, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 400, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9539170506912442, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9605568445475638, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9572254335260115, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03944315545243619, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9216589861751152, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9280742459396751, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9248554913294798, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9539170506912442, "step_test_time": 4.3251071340055205, "step_plot_time": 3.0468288880074397, "step_validation_cycle_time": 10.697668223001529} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..268d2c24b61e3e1852d08ff416b25e7c13365e83 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 0.6833767294883728, "Classification": 0.0007696592947468162, "Regression_pt": 0.04526966065168381, "Regression_eta": 0.0110852075740695, "Regression_sin_phi": 0.006350328680127859, "Regression_cos_phi": 0.004911704920232296, "Regression_energy": 0.04315624013543129, "Total": 1.5656156539916992, "Total_unweighted": 0.7949195307446644}, "train_group": {}, "valid": {"Classification_binary": 2.7019779682159424, "Classification": 0.01774994656443596, "Regression_pt": 0.1451350301504135, "Regression_eta": 0.03620944544672966, "Regression_sin_phi": 0.007384653203189373, "Regression_cos_phi": 0.007830381393432617, "Regression_energy": 0.1494140475988388, "Total": 5.599095821380615, "Total_unweighted": 3.0657014725729823, "metrics/particle/class_1/efficiency": 0.88453608751297, "metrics/particle/class_1/pid_accuracy": 0.9673659801483154, "metrics/particle/class_2/efficiency": 0.1818181872367859, "metrics/particle/class_2/pid_accuracy": 0.75, "metrics/particle/class_3/efficiency": 0.6546489596366882, "metrics/particle/class_3/pid_accuracy": 0.9826086759567261, "metrics/particle/class_4/efficiency": 0.8541666865348816, "metrics/particle/class_4/pid_accuracy": 0.9024389982223511, "metrics/particle/class_5/efficiency": 0.9878048896789551, "metrics/particle/class_5/pid_accuracy": 0.9876543283462524, "metrics/particle/count/bias_mean": -0.08799999952316284, "metrics/particle/count/mae": 0.6159999966621399, "metrics/particle/count/prediction_mean": 4.760000228881836, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.09216712415218353, "metrics/particle/event/energy_response_mean": 1.014825701713562, "metrics/particle/event/met_abs_error": 2.299906015396118, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0878487378358841, "metrics/particle/event/scalar_pt_response_mean": 1.016690731048584, "metrics/particle/event/vector_pt_closure": 0.0970926284790039, "metrics/particle/matched/delta_eta_abs_mean": 0.024247970432043076, "metrics/particle/matched/delta_phi_abs_mean": 0.02047739364206791, "metrics/particle/matched/delta_r_mean": 0.03544759377837181, "metrics/particle/matched/energy_relative_abs_mean": 0.08088289201259613, "metrics/particle/matched/pid_accuracy": 0.9681190252304077, "metrics/particle/matched/pt_relative_abs_mean": 0.08075613528490067, "metrics/particle/matching/duplicate_fraction": 0.02521008439362049, "metrics/particle/matching/efficiency": 0.7764026522636414, "metrics/particle/matching/f1": 0.7835137248039246, "metrics/particle/matching/purity": 0.7907562851905823, "metrics/particle/matching/target_coverage_dr0p05": 0.6146864891052246, "metrics/particle/matching/target_coverage_dr0p10": 0.801980197429657, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7392739057540894, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8547855019569397}, "step_train_time": 0.5187466144561768, "step_valid_time": 2.8651236439472996, "step_total_time": 3229.897052526474, "elapsed_training_loop_seconds": 3229.897052526474, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0144366951863715, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05973127356881314, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 435, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 410, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 402, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9447004608294931, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9425287356321839, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9436133486766398, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05747126436781609, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9262672811059908, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9241379310344827, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9252013808975834, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9447004608294931, "step_test_time": 4.300997265963815, "step_plot_time": 3.0937253680313006, "step_validation_cycle_time": 10.688915023987647} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..b62b53efa962ad0438f2e42249f110fa201d413a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 0.37375521659851074, "Classification": 0.00238040485419333, "Regression_pt": 0.0845857784152031, "Regression_eta": 0.009669068269431591, "Regression_sin_phi": 0.005816029384732246, "Regression_cos_phi": 0.004938990343362093, "Regression_energy": 0.08510468155145645, "Total": 1.8947539329528809, "Total_unweighted": 0.5662501694168895}, "train_group": {}, "valid": {"Classification_binary": 4.246802806854248, "Classification": 0.027661681175231934, "Regression_pt": 0.1738741546869278, "Regression_eta": 0.03320179134607315, "Regression_sin_phi": 0.007073991000652313, "Regression_cos_phi": 0.007510598283261061, "Regression_energy": 0.17279697954654694, "Total": 7.65954065322876, "Total_unweighted": 4.668922002892941, "metrics/particle/class_1/efficiency": 0.88453608751297, "metrics/particle/class_1/pid_accuracy": 0.9766899943351746, "metrics/particle/class_2/efficiency": 0.22727273404598236, "metrics/particle/class_2/pid_accuracy": 0.6000000238418579, "metrics/particle/class_3/efficiency": 0.6375711560249329, "metrics/particle/class_3/pid_accuracy": 0.9910714030265808, "metrics/particle/class_4/efficiency": 0.8645833134651184, "metrics/particle/class_4/pid_accuracy": 0.9036144614219666, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.9878048896789551, "metrics/particle/count/bias_mean": -0.012000000104308128, "metrics/particle/count/mae": 0.5960000157356262, "metrics/particle/count/prediction_mean": 4.835999965667725, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.09309908002614975, "metrics/particle/event/energy_response_mean": 0.9878209233283997, "metrics/particle/event/met_abs_error": 2.602536678314209, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09157252311706543, "metrics/particle/event/scalar_pt_response_mean": 0.9894906878471375, "metrics/particle/event/vector_pt_closure": 0.10486642271280289, "metrics/particle/matched/delta_eta_abs_mean": 0.022760653868317604, "metrics/particle/matched/delta_phi_abs_mean": 0.018792476505041122, "metrics/particle/matched/delta_r_mean": 0.03293704241514206, "metrics/particle/matched/energy_relative_abs_mean": 0.07626444101333618, "metrics/particle/matched/pid_accuracy": 0.9743315577507019, "metrics/particle/matched/pt_relative_abs_mean": 0.07579599320888519, "metrics/particle/matching/duplicate_fraction": 0.032258063554763794, "metrics/particle/matching/efficiency": 0.7714521288871765, "metrics/particle/matching/f1": 0.7724080681800842, "metrics/particle/matching/purity": 0.773366391658783, "metrics/particle/matching/target_coverage_dr0p05": 0.6237623691558838, "metrics/particle/matching/target_coverage_dr0p10": 0.7970296740531921, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7334983348846436, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8473597168922424}, "step_train_time": 0.5273571014404297, "step_valid_time": 2.820704870042391, "step_total_time": 4311.351669311523, "elapsed_training_loop_seconds": 4311.351669311523, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9934057675384198, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06550231222573077, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 427, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 409, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 396, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9423963133640553, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9578454332552693, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9500580720092915, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04215456674473068, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9124423963133641, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.927400468384075, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9198606271777003, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9423963133640553, "step_test_time": 4.723170542041771, "step_plot_time": 3.0944406559574418, "step_validation_cycle_time": 11.065568878955673} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..da1145a06aa3359862f8d75b6bf0ddce70bf8b5d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 0.42387616634368896, "Classification": 0.002238782588392496, "Regression_pt": 0.04234817251563072, "Regression_eta": 0.003866266692057252, "Regression_sin_phi": 0.0023220237344503403, "Regression_cos_phi": 0.0019885345827788115, "Regression_energy": 0.044972825795412064, "Total": 1.1965491771697998, "Total_unweighted": 0.5216127722524107}, "train_group": {}, "valid": {"Classification_binary": 5.951561450958252, "Classification": 0.032959338277578354, "Regression_pt": 0.1823326200246811, "Regression_eta": 0.036366820335388184, "Regression_sin_phi": 0.00783283356577158, "Regression_cos_phi": 0.007242303341627121, "Regression_energy": 0.18058809638023376, "Total": 9.58366584777832, "Total_unweighted": 6.398883462883532, "metrics/particle/class_1/efficiency": 0.8907216787338257, "metrics/particle/class_1/pid_accuracy": 0.9699074029922485, "metrics/particle/class_2/efficiency": 0.1818181872367859, "metrics/particle/class_2/pid_accuracy": 0.5, "metrics/particle/class_3/efficiency": 0.622390866279602, "metrics/particle/class_3/pid_accuracy": 0.9817073345184326, "metrics/particle/class_4/efficiency": 0.8541666865348816, "metrics/particle/class_4/pid_accuracy": 0.9146341681480408, "metrics/particle/class_5/efficiency": 0.9878048896789551, "metrics/particle/class_5/pid_accuracy": 1.0, "metrics/particle/count/bias_mean": 0.07999999821186066, "metrics/particle/count/mae": 0.6159999966621399, "metrics/particle/count/prediction_mean": 4.927999973297119, "metrics/particle/count/target_mean": 4.8480000495910645, "metrics/particle/event/energy_relative_abs_error": 0.09127882122993469, "metrics/particle/event/energy_response_mean": 0.9934391975402832, "metrics/particle/event/met_abs_error": 2.4376978874206543, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08903326839208603, "metrics/particle/event/scalar_pt_response_mean": 0.9941374659538269, "metrics/particle/event/vector_pt_closure": 0.10187649726867676, "metrics/particle/matched/delta_eta_abs_mean": 0.02300499565899372, "metrics/particle/matched/delta_phi_abs_mean": 0.02032766304910183, "metrics/particle/matched/delta_r_mean": 0.03420731797814369, "metrics/particle/matched/energy_relative_abs_mean": 0.07222449779510498, "metrics/particle/matched/pid_accuracy": 0.9697950482368469, "metrics/particle/matched/pt_relative_abs_mean": 0.07176696509122849, "metrics/particle/matching/duplicate_fraction": 0.025974025949835777, "metrics/particle/matching/efficiency": 0.7648515105247498, "metrics/particle/matching/f1": 0.7585924863815308, "metrics/particle/matching/purity": 0.7524350881576538, "metrics/particle/matching/target_coverage_dr0p05": 0.6138613820075989, "metrics/particle/matching/target_coverage_dr0p10": 0.7945544719696045, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7425742745399475, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8547855019569397}, "step_train_time": 0.5262393951416016, "step_valid_time": 2.842782622028608, "step_total_time": 5392.843901395798, "elapsed_training_loop_seconds": 5392.843901395798, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9951770951036355, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0504679875090448, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 434, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 430, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 407, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 395, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9377880184331797, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9465116279069767, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9421296296296297, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053488372093023255, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9101382488479263, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9186046511627907, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9143518518518519, "step/cld_edm_z_tautau_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9377880184331797, "step_test_time": 4.401908691972494, "step_plot_time": 3.0668762259883806, "step_validation_cycle_time": 10.727593862975482} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..000910a574fa437894ab8febdf040ac72488efa9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_5p1mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_z_tautau_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_z_tautau_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8e180e5b9a48ef5fba61802ff7e6e1c8a1991e98 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:128a1f47a5390bead05bf617720fc08bfb93f4aad55a3ca30d7061c1efc266c1 +size 4404 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b3e85758a2a3ff7ba368b6914d9599d702a5058f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..0ff3cd85f0a57290c74dbd13399c4a331c82ef7d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6d92e1fa18330362b6acd5e4ae545f59a05456a4 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..9cb5bec44d9c9524d005d700a290d7a3e88e1c04 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f2274ff3572a5f3395cff4050c841762b8aa906a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..4cd2c43b1fc1539a9d8ac47e4c413dd07e393b5d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a141aec60def0204af23bb438411b36dd890954b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..9a710693ae535a9761d0627b0136349e62d4f313 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dac701bcbfd21656ab3dfab978757d5826407da9 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..ca1c49f223c0565666015ead3be4f7224c081934 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8e0e3fd54988033ec9010a858984943f34182d48 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..a841d54a2613a4ec4a08361bedfb6330443191d4 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/plots_step_50000/cld_edm_z_tautau_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/runs/train/events.out.tfevents.1794648991.workergpu164.25375.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/runs/train/events.out.tfevents.1794648991.workergpu164.25375.0 new file mode 100644 index 0000000000000000000000000000000000000000..62b81fc9fd5d19075506fb172f1bf4dcb0cadfc3 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/runs/train/events.out.tfevents.1794648991.workergpu164.25375.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7cbe6ec3b91addb378ea1ba13ce5ab3e6e63e0355df162da321f9546073092d +size 1843423 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/runs/valid/events.out.tfevents.1794648991.workergpu164.25375.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/runs/valid/events.out.tfevents.1794648991.workergpu164.25375.1 new file mode 100644 index 0000000000000000000000000000000000000000..4e63c56f606d5e116ee3690c552dc06da320e5d8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/runs/valid/events.out.tfevents.1794648991.workergpu164.25375.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79964bbd0b05e83d04269738dffae920abfe3920e3d5ca15f1bed8c41d2e7896 +size 2284094 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..e8c27a25f6c1e5d06e8d64c50a431abea1631150 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_ecal_5p1mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_5p1mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_5p1mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_z_tautau_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_5p1mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_z_tautau_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..405be2906fc07796e70760554b869fef8678df21 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/ecal_5p1mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_z_tautau_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_z_tautau_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_z_tautau_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8d8dbe785e95231ed18f30eb68ef75211ad01b36 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/train.log @@ -0,0 +1,999 @@ +rank0 [2026-11-14 04:36:27,925] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-14 04:36:27,925] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 04:36:28,427] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 04:36:28,428] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 04:36:28,428] INFO : Instantiating model +rank0 [2026-11-14 04:36:28,450] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 04:36:28,450] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 04:36:28,452] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 04:36:28,452] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 04:36:28,452] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 04:36:28,452] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 04:36:28,452] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,455] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 04:36:28,455] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,458] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 04:36:28,458] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,460] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 04:36:28,461] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,463] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 04:36:28,463] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,466] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 04:36:28,466] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,469] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 04:36:28,469] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 04:36:28,469] INFO : Initializing output DNNs +rank0 [2026-11-14 04:36:28,470] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,473] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,482] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 04:36:28,482] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 04:36:28,482] INFO : jagged_attention enabled=True +rank0 [2026-11-14 04:36:28,482] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 04:36:28,482] INFO : task_queries enabled=True +rank0 [2026-11-14 04:36:28,482] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 04:36:28,482] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 04:36:28,482] INFO : classification_query parameters: 256 +rank0 [2026-11-14 04:36:28,482] INFO : regression_query parameters: 256 +rank0 [2026-11-14 04:36:28,482] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 04:36:28,482] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 04:36:28,482] INFO : output_mode=elementwise +rank0 [2026-11-14 04:36:28,482] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 04:36:28,482] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 04:36:28,482] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 04:36:28,482] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 04:36:28,483] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 04:36:28,483] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 04:36:28,483] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 04:36:28,483] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 04:36:28,484] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 04:36:28,484] INFO : MLPF __init__ done +rank0 [2026-11-14 04:36:28,484] INFO : Instantiated model +rank0 [2026-11-14 04:36:28,485] INFO : Moving model to device rank=0 +rank0 [2026-11-14 04:36:28,495] INFO : Moved model to device rank=0 +rank0 [2026-11-14 04:36:28,496] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 04:36:28,592] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 04:36:28,592] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 04:36:28,594] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 04:36:28,594] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 04:36:28,594] INFO : Total parameters: 3910930 +rank0 [2026-11-14 04:36:28,596] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 04:36:28,599] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596 +rank0 [2026-11-14 04:36:28,599] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596 +rank0 [2026-11-14 04:36:28,599] INFO : Getting dataloaders +rank0 [2026-11-14 04:36:28,599] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:36:29,927] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:29,969] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:36:30,009] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:36:30,064] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,121] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,532] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,587] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,639] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,684] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:36:30,722] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,722] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 04:36:30,722] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 04:36:30,722] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 04:36:30,722] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:36:30,734] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,747] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,757] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,768] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,779] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,792] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,804] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,815] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,827] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,840] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,840] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 04:36:30,840] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 04:36:30,888] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:36:31,034] INFO : Got dataloaders +rank0 [2026-11-14 04:36:31,034] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 04:36:31,034] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 04:36:31,035] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 04:36:31,040] INFO : Creating train_iterator +rank0 [2026-11-14 04:36:31,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:31,071] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:36:31,290] INFO : Created train_iterator +rank0 [2026-11-14 04:36:43,204] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.40914969921112, 'Classification': 0.5980110427737236, 'Regression_pt': 0.7724759268760681, 'Regression_eta': 1.1418460792303085, 'Regression_sin_phi': 0.7991263212263584, 'Regression_cos_phi': 0.813458954244852, 'Regression_energy': 1.2159733620285988}, weights={'Classification_binary': 1.0, 'Classification': 12.389653205871582, 'Regression_pt': 9.591431617736816, 'Regression_eta': 6.488746643066406, 'Regression_sin_phi': 9.271562576293945, 'Regression_cos_phi': 9.108203887939453, 'Regression_energy': 6.093183994293213} +rank0 [2026-11-14 04:37:32,778] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 04:37:32,778] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:38:19,749] INFO : Step 1000/500000 rank0 | Train Loss: 8.8077 | Train Loss Unweighted: 4.5601 | Train Loss Components: Classification: 0.0803 | Classification_binary: 4.0058 | Regression_cos_phi: 0.0553 | Regression_energy: 0.1631 | Regression_eta: 0.0423 | Regression_pt: 0.1777 | Regression_sin_phi: 0.0355 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.9954 | Classification_binary: 4.0058 | Regression_cos_phi: 0.5041 | Regression_energy: 0.9940 | Regression_eta: 0.2747 | Regression_pt: 1.7042 | Regression_sin_phi: 0.3296 | LR: 2.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6007s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0648 | max_tensorboard_logging: 0.2241 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2559.2 | uss_MB: 1340.5 | child_rss_MB: 22496.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:38:32,874] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:38:32,874] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:32,297] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:39:32,297] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:06,202] INFO : Step 2000/500000 rank0 | Train Loss: 8.7654 | Train Loss Unweighted: 3.8513 | Train Loss Components: Classification: 0.0585 | Classification_binary: 3.1757 | Regression_cos_phi: 0.0170 | Regression_energy: 0.2721 | Regression_eta: 0.0290 | Regression_pt: 0.2848 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.7251 | Classification_binary: 3.1757 | Regression_cos_phi: 0.1545 | Regression_energy: 1.6577 | Regression_eta: 0.1880 | Regression_pt: 2.7316 | Regression_sin_phi: 0.1328 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5730s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0027 | max_backward: 0.0383 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1994 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2559.9 | uss_MB: 1341.3 | child_rss_MB: 22489.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:40:06,204] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:40:32,627] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:40:32,627] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:32,678] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:41:32,678] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:53,419] INFO : Step 3000/500000 rank0 | Train Loss: 7.0901 | Train Loss Unweighted: 3.3502 | Train Loss Components: Classification: 0.0940 | Classification_binary: 2.8941 | Regression_cos_phi: 0.1078 | Regression_energy: 0.0792 | Regression_eta: 0.0279 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0815 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 1.1640 | Classification_binary: 2.8941 | Regression_cos_phi: 0.9818 | Regression_energy: 0.4827 | Regression_eta: 0.1809 | Regression_pt: 0.6308 | Regression_sin_phi: 0.7557 | LR: 2.05e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5709s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1973 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2563.2 | uss_MB: 1344.9 | child_rss_MB: 22489.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:42:32,945] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:42:32,945] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:32,683] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:43:32,683] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:40,178] INFO : Step 4000/500000 rank0 | Train Loss: 6.7464 | Train Loss Unweighted: 3.9315 | Train Loss Components: Classification: 0.0468 | Classification_binary: 3.5623 | Regression_cos_phi: 0.0173 | Regression_energy: 0.1228 | Regression_eta: 0.0136 | Regression_pt: 0.1434 | Regression_sin_phi: 0.0253 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.5801 | Classification_binary: 3.5623 | Regression_cos_phi: 0.1578 | Regression_energy: 0.7483 | Regression_eta: 0.0883 | Regression_pt: 1.3750 | Regression_sin_phi: 0.2346 | LR: 2.08e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5974s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0339 | max_loss: 0.0024 | max_backward: 0.0378 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.2266 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2563.2 | uss_MB: 1344.9 | child_rss_MB: 22489.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:43:40,180] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:44:32,761] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:44:32,762] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:27,192] INFO : Step 5000/500000 rank0 | Train Loss: 7.9320 | Train Loss Unweighted: 3.2771 | Train Loss Components: Classification: 0.0128 | Classification_binary: 2.6180 | Regression_cos_phi: 0.0064 | Regression_energy: 0.2855 | Regression_eta: 0.0127 | Regression_pt: 0.3339 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1590 | Classification_binary: 2.6180 | Regression_cos_phi: 0.0581 | Regression_energy: 1.7395 | Regression_eta: 0.0823 | Regression_pt: 3.2024 | Regression_sin_phi: 0.0727 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5407s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0387 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1752 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2561.1 | uss_MB: 1342.8 | child_rss_MB: 22489.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:45:33,376] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:45:33,376] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:32,760] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:46:32,760] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:13,246] INFO : Step 6000/500000 rank0 | Train Loss: 8.4547 | Train Loss Unweighted: 3.5909 | Train Loss Components: Classification: 0.0100 | Classification_binary: 2.8883 | Regression_cos_phi: 0.0062 | Regression_energy: 0.3349 | Regression_eta: 0.0077 | Regression_pt: 0.3378 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1236 | Classification_binary: 2.8883 | Regression_cos_phi: 0.0563 | Regression_energy: 2.0404 | Regression_eta: 0.0497 | Regression_pt: 3.2403 | Regression_sin_phi: 0.0560 | LR: 2.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0341 | max_loss: 0.0026 | max_backward: 0.0419 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1976 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2561.1 | uss_MB: 1342.8 | child_rss_MB: 22489.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:47:13,248] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:47:32,474] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:47:32,474] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:31,772] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:48:31,773] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:59,500] INFO : Step 7000/500000 rank0 | Train Loss: 2.6730 | Train Loss Unweighted: 1.7274 | Train Loss Components: Classification: 0.0235 | Classification_binary: 1.6060 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0344 | Regression_eta: 0.0120 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2908 | Classification_binary: 1.6060 | Regression_cos_phi: 0.0666 | Regression_energy: 0.2094 | Regression_eta: 0.0779 | Regression_pt: 0.3730 | Regression_sin_phi: 0.0494 | LR: 2.26e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5776s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0659 | max_tensorboard_logging: 0.2030 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2561.9 | uss_MB: 1343.6 | child_rss_MB: 22488.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:49:32,121] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:49:32,121] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:31,481] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:50:31,481] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:46,044] INFO : Step 8000/500000 rank0 | Train Loss: 3.5124 | Train Loss Unweighted: 1.6850 | Train Loss Components: Classification: 0.0275 | Classification_binary: 1.4560 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0332 | Regression_eta: 0.0163 | Regression_pt: 0.0293 | Regression_sin_phi: 0.0569 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.3403 | Classification_binary: 1.4560 | Regression_cos_phi: 0.5998 | Regression_energy: 0.2023 | Regression_eta: 0.1057 | Regression_pt: 0.2810 | Regression_sin_phi: 0.5274 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5761s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0376 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.2107 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2561.9 | uss_MB: 1343.7 | child_rss_MB: 22488.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:50:46,046] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:50:46,117] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:51:31,865] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:51:31,865] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:31,589] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:52:31,589] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:32,930] INFO : Step 9000/500000 rank0 | Train Loss: 5.5896 | Train Loss Unweighted: 3.0148 | Train Loss Components: Classification: 0.0050 | Classification_binary: 2.6483 | Regression_cos_phi: 0.0154 | Regression_energy: 0.1529 | Regression_eta: 0.0139 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 2.6483 | Regression_cos_phi: 0.1403 | Regression_energy: 0.9317 | Regression_eta: 0.0899 | Regression_pt: 1.6651 | Regression_sin_phi: 0.0519 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5686s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0379 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.2062 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2560.1 | uss_MB: 1341.9 | child_rss_MB: 22490.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:53:32,096] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:53:32,099] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:20,112] INFO : Step 10000/500000 rank0 | Train Loss: 5.0393 | Train Loss Unweighted: 2.6469 | Train Loss Components: Classification: 0.0124 | Classification_binary: 2.3063 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1545 | Regression_eta: 0.0080 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1538 | Classification_binary: 2.3063 | Regression_cos_phi: 0.0460 | Regression_energy: 0.9412 | Regression_eta: 0.0521 | Regression_pt: 1.4942 | Regression_sin_phi: 0.0457 | LR: 2.52e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6434s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0375 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.2650 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2560.7 | uss_MB: 1342.4 | child_rss_MB: 22490.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:54:20,114] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:54:20,185] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:54:20,186] INFO : Running validation on rank0 +rank0 [2026-11-14 04:54:20,186] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:54:20,186] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:54:20,351] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:54:20,525] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:54:20,592] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:54:20,644] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:54:20,699] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:54:20,744] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:54:20,797] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:54:20,850] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:54:20,897] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:54:20,961] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:54:21,026] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:54:21,079] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:54:21,134] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:54:21,178] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:54:21,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:54:21,285] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:54:21,334] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:54:21,337] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:54:21,340] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:54:21,342] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:54:21,343] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:54:21,346] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:54:21,348] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:54:21,350] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:54:21,352] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:54:21,353] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:54:21,355] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:54:21,356] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:54:21,358] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:54:21,360] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:54:21,362] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:54:21,364] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:54:22,833] INFO : VALIDATION | Step=10000/500000 | Train Loss=5.0393 | Valid Loss=6.0359 | Train Loss Unweighted=2.6469 | Valid Loss Unweighted=2.9008 | Stale=0 | ETA=875.3m +rank0 [2026-11-14 04:54:22,834] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:54:22,948] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:54:22,956] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:23,018] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:54:23,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:54:23,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:54:24,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:54:24,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:54:24,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:54:24,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:54:25,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:54:25,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:54:25,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:54:25,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:54:26,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:54:26,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:54:26,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:54:26,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:54:27,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:54:27,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:54:27,350] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:54:27,358] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:27,883] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:28,588] INFO : Plotted number of elements +rank0 [2026-11-14 04:54:29,859] INFO : Plotted jets +rank0 [2026-11-14 04:54:30,792] INFO : Plotted jet ratio +rank0 [2026-11-14 04:54:43,496] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:54:43,496] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:55:43,259] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:55:43,259] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:17,954] INFO : Step 11000/500000 rank0 | Train Loss: 4.7022 | Train Loss Unweighted: 3.0012 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.7658 | Regression_cos_phi: 0.0052 | Regression_energy: 0.0974 | Regression_eta: 0.0099 | Regression_pt: 0.0958 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2399 | Classification_binary: 2.7658 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5933 | Regression_eta: 0.0640 | Regression_pt: 0.9188 | Regression_sin_phi: 0.0734 | LR: 2.63e-05 | DataLoad Time: 0.0028s | Model Forward Time: 0.6073s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0410 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.2278 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2683.5 | uss_MB: 1472.3 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:56:43,394] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:56:43,394] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:57:42,672] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:57:42,672] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:04,117] INFO : Step 12000/500000 rank0 | Train Loss: 4.5926 | Train Loss Unweighted: 2.1270 | Train Loss Components: Classification: 0.0121 | Classification_binary: 1.7757 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1579 | Regression_eta: 0.0097 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1504 | Classification_binary: 1.7757 | Regression_cos_phi: 0.0413 | Regression_energy: 0.9618 | Regression_eta: 0.0628 | Regression_pt: 1.5394 | Regression_sin_phi: 0.0612 | LR: 2.75e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5706s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0340 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1847 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2683.5 | uss_MB: 1472.3 | child_rss_MB: 22490.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:58:04,119] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:58:04,191] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:58:42,542] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:58:42,542] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:42,219] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:59:42,219] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:50,600] INFO : Step 13000/500000 rank0 | Train Loss: 3.7013 | Train Loss Unweighted: 1.9955 | Train Loss Components: Classification: 0.0233 | Classification_binary: 1.7637 | Regression_cos_phi: 0.0099 | Regression_energy: 0.0846 | Regression_eta: 0.0155 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2881 | Classification_binary: 1.7637 | Regression_cos_phi: 0.0906 | Regression_energy: 0.5152 | Regression_eta: 0.1004 | Regression_pt: 0.8802 | Regression_sin_phi: 0.0631 | LR: 2.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6099s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0360 | max_loss: 0.0025 | max_backward: 0.0375 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.2073 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2632.1 | uss_MB: 1420.9 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:00:42,380] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:00:42,380] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:37,169] INFO : Step 14000/500000 rank0 | Train Loss: 2.9774 | Train Loss Unweighted: 1.7265 | Train Loss Components: Classification: 0.0048 | Classification_binary: 1.5469 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0825 | Regression_eta: 0.0044 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 1.5469 | Regression_cos_phi: 0.0531 | Regression_energy: 0.5028 | Regression_eta: 0.0285 | Regression_pt: 0.7710 | Regression_sin_phi: 0.0161 | LR: 3.02e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5163s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0377 | max_optimizer_step: 0.0647 | max_tensorboard_logging: 0.1457 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2632.8 | uss_MB: 1421.6 | child_rss_MB: 22490.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:01:37,171] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:01:37,240] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:01:42,540] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 05:01:42,540] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:42,256] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 05:02:42,257] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:24,400] INFO : Step 15000/500000 rank0 | Train Loss: 4.5850 | Train Loss Unweighted: 2.9683 | Train Loss Components: Classification: 0.0122 | Classification_binary: 2.7415 | Regression_cos_phi: 0.0062 | Regression_energy: 0.0939 | Regression_eta: 0.0105 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1517 | Classification_binary: 2.7415 | Regression_cos_phi: 0.0565 | Regression_energy: 0.5722 | Regression_eta: 0.0684 | Regression_pt: 0.9436 | Regression_sin_phi: 0.0512 | LR: 3.17e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0391 | max_optimizer_step: 0.0765 | max_tensorboard_logging: 0.1767 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2637.2 | uss_MB: 1426.0 | child_rss_MB: 22490.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:03:42,858] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 05:03:42,858] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:42,412] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 05:04:42,412] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:10,914] INFO : Step 16000/500000 rank0 | Train Loss: 3.9743 | Train Loss Unweighted: 2.3588 | Train Loss Components: Classification: 0.0034 | Classification_binary: 2.1258 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1079 | Regression_eta: 0.0042 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 2.1258 | Regression_cos_phi: 0.0655 | Regression_energy: 0.6575 | Regression_eta: 0.0275 | Regression_pt: 1.0183 | Regression_sin_phi: 0.0369 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.6023s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0374 | max_optimizer_step: 0.0771 | max_tensorboard_logging: 0.2102 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2637.3 | uss_MB: 1426.1 | child_rss_MB: 22489.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:05:10,916] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:05:10,984] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:05:42,622] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 05:05:42,622] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:42,357] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 05:06:42,357] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:57,711] INFO : Step 17000/500000 rank0 | Train Loss: 3.8166 | Train Loss Unweighted: 2.5697 | Train Loss Components: Classification: 0.0080 | Classification_binary: 2.3963 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0670 | Regression_eta: 0.0077 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0146 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0993 | Classification_binary: 2.3963 | Regression_cos_phi: 0.0386 | Regression_energy: 0.4080 | Regression_eta: 0.0497 | Regression_pt: 0.6892 | Regression_sin_phi: 0.1355 | LR: 3.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5717s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0374 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1918 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2632.0 | uss_MB: 1420.8 | child_rss_MB: 22489.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:07:42,892] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 05:07:42,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:42,636] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 05:08:42,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:44,816] INFO : Step 18000/500000 rank0 | Train Loss: 3.6751 | Train Loss Unweighted: 1.5132 | Train Loss Components: Classification: 0.0106 | Classification_binary: 1.2169 | Regression_cos_phi: 0.0044 | Regression_energy: 0.1062 | Regression_eta: 0.0122 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1309 | Classification_binary: 1.2169 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6473 | Regression_eta: 0.0791 | Regression_pt: 1.5062 | Regression_sin_phi: 0.0549 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5743s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0375 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1956 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2632.8 | uss_MB: 1421.6 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:08:44,818] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:08:44,889] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:09:42,890] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 05:09:42,890] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:31,506] INFO : Step 19000/500000 rank0 | Train Loss: 2.9176 | Train Loss Unweighted: 1.6474 | Train Loss Components: Classification: 0.0170 | Classification_binary: 1.4739 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0714 | Regression_eta: 0.0046 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2102 | Classification_binary: 1.4739 | Regression_cos_phi: 0.0387 | Regression_energy: 0.4349 | Regression_eta: 0.0301 | Regression_pt: 0.6964 | Regression_sin_phi: 0.0334 | LR: 3.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5904s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0369 | max_loss: 0.0029 | max_backward: 0.0392 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1839 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2637.0 | uss_MB: 1425.9 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:10:42,909] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 05:10:42,910] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:42,738] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 05:11:42,738] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:18,081] INFO : Step 20000/500000 rank0 | Train Loss: 1.3325 | Train Loss Unweighted: 0.6857 | Train Loss Components: Classification: 0.0087 | Classification_binary: 0.5977 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0348 | Regression_eta: 0.0031 | Regression_pt: 0.0365 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1079 | Classification_binary: 0.5977 | Regression_cos_phi: 0.0246 | Regression_energy: 0.2122 | Regression_eta: 0.0201 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0200 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5670s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0643 | max_tensorboard_logging: 0.1982 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2637.4 | uss_MB: 1426.2 | child_rss_MB: 22490.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:12:18,083] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:12:18,151] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:12:18,152] INFO : Running validation on rank0 +rank0 [2026-11-14 05:12:18,152] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:12:18,153] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:12:18,271] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 05:12:18,377] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:12:18,475] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:12:18,542] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:12:18,606] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:12:18,670] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:12:18,735] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:12:18,801] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:12:18,900] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:12:19,000] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:12:19,101] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:12:19,170] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:12:19,234] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:12:19,299] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:12:19,363] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:12:19,426] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:12:19,533] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:12:19,537] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:12:19,541] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:12:19,543] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:12:19,545] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:12:19,547] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:12:19,549] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:12:19,551] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:12:19,553] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:12:19,555] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:12:19,557] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:12:19,558] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:12:19,559] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:12:19,561] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:12:19,563] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:12:19,564] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:12:21,041] INFO : VALIDATION | Step=20000/500000 | Train Loss=1.3325 | Valid Loss=5.3294 | Train Loss Unweighted=0.6857 | Valid Loss Unweighted=2.6964 | Stale=0 | ETA=860.0m +rank0 [2026-11-14 05:12:21,042] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:12:21,140] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:12:21,147] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:12:21,208] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:12:21,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:12:21,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:12:22,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:12:22,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:12:22,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:12:22,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:12:23,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:12:23,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:12:23,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:12:23,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:12:24,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:12:24,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:12:24,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:12:24,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:12:25,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:12:25,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:12:25,365] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:12:25,372] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:12:25,895] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:12:26,282] INFO : Plotted number of elements +rank0 [2026-11-14 05:12:27,477] INFO : Plotted jets +rank0 [2026-11-14 05:12:28,407] INFO : Plotted jet ratio +rank0 [2026-11-14 05:12:53,420] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 05:12:53,420] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:52,902] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 05:13:52,902] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:14:15,261] INFO : Step 21000/500000 rank0 | Train Loss: 4.2539 | Train Loss Unweighted: 2.0038 | Train Loss Components: Classification: 0.0094 | Classification_binary: 1.6809 | Regression_cos_phi: 0.0037 | Regression_energy: 0.1524 | Regression_eta: 0.0049 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1161 | Classification_binary: 1.6809 | Regression_cos_phi: 0.0335 | Regression_energy: 0.9283 | Regression_eta: 0.0317 | Regression_pt: 1.4431 | Regression_sin_phi: 0.0203 | LR: 4.28e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6420s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0342 | max_loss: 0.0025 | max_backward: 0.0378 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.2521 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2665.0 | uss_MB: 1453.8 | child_rss_MB: 22491.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:14:52,918] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 05:14:52,918] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:52,314] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 05:15:52,314] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:01,558] INFO : Step 22000/500000 rank0 | Train Loss: 2.9036 | Train Loss Unweighted: 1.2899 | Train Loss Components: Classification: 0.0030 | Classification_binary: 1.0606 | Regression_cos_phi: 0.0082 | Regression_energy: 0.0936 | Regression_eta: 0.0100 | Regression_pt: 0.1082 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 1.0606 | Regression_cos_phi: 0.0751 | Regression_energy: 0.5704 | Regression_eta: 0.0649 | Regression_pt: 1.0375 | Regression_sin_phi: 0.0583 | LR: 4.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6443s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0382 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2435 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2666.0 | uss_MB: 1454.8 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:16:01,560] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:16:01,629] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:16:52,963] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 05:16:52,963] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:48,626] INFO : Step 23000/500000 rank0 | Train Loss: 3.4638 | Train Loss Unweighted: 1.9056 | Train Loss Components: Classification: 0.0197 | Classification_binary: 1.6909 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0931 | Regression_eta: 0.0042 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2447 | Classification_binary: 1.6909 | Regression_cos_phi: 0.0264 | Regression_energy: 0.5675 | Regression_eta: 0.0275 | Regression_pt: 0.8747 | Regression_sin_phi: 0.0321 | LR: 4.73e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.6491s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0379 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.2637 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.1 | uss_MB: 1457.9 | child_rss_MB: 22490.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:17:53,083] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 05:17:53,083] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:52,422] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 05:18:52,422] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:34,732] INFO : Step 24000/500000 rank0 | Train Loss: 2.2348 | Train Loss Unweighted: 1.4395 | Train Loss Components: Classification: 0.0048 | Classification_binary: 1.3260 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0516 | Regression_eta: 0.0035 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 1.3260 | Regression_cos_phi: 0.0266 | Regression_energy: 0.3146 | Regression_eta: 0.0230 | Regression_pt: 0.4571 | Regression_sin_phi: 0.0276 | LR: 4.97e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5764s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0343 | max_loss: 0.0026 | max_backward: 0.0376 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1896 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.1 | uss_MB: 1457.9 | child_rss_MB: 22490.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:19:34,734] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:19:34,801] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:19:52,328] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 05:19:52,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:51,844] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 05:20:51,844] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:21,408] INFO : Step 25000/500000 rank0 | Train Loss: 7.9381 | Train Loss Unweighted: 2.9835 | Train Loss Components: Classification: 0.0137 | Classification_binary: 2.2697 | Regression_cos_phi: 0.0093 | Regression_energy: 0.3387 | Regression_eta: 0.0078 | Regression_pt: 0.3349 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1698 | Classification_binary: 2.2697 | Regression_cos_phi: 0.0846 | Regression_energy: 2.0636 | Regression_eta: 0.0509 | Regression_pt: 3.2118 | Regression_sin_phi: 0.0878 | LR: 5.22e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5996s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0343 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.1933 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.1 | uss_MB: 1453.9 | child_rss_MB: 22488.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:21:52,293] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 05:21:52,293] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:52,204] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 05:22:52,204] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:23:08,434] INFO : Step 26000/500000 rank0 | Train Loss: 3.2594 | Train Loss Unweighted: 1.0386 | Train Loss Components: Classification: 0.0068 | Classification_binary: 0.7194 | Regression_cos_phi: 0.0033 | Regression_energy: 0.1494 | Regression_eta: 0.0050 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0839 | Classification_binary: 0.7194 | Regression_cos_phi: 0.0298 | Regression_energy: 0.9102 | Regression_eta: 0.0327 | Regression_pt: 1.4380 | Regression_sin_phi: 0.0453 | LR: 5.47e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5989s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.2022 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.9 | uss_MB: 1454.7 | child_rss_MB: 22492.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:23:08,436] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:23:08,507] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:23:52,476] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 05:23:52,477] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:52,303] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 05:24:52,303] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:55,384] INFO : Step 27000/500000 rank0 | Train Loss: 2.7915 | Train Loss Unweighted: 1.9927 | Train Loss Components: Classification: 0.0085 | Classification_binary: 1.8815 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0449 | Regression_eta: 0.0067 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1050 | Classification_binary: 1.8815 | Regression_cos_phi: 0.0233 | Regression_energy: 0.2734 | Regression_eta: 0.0433 | Regression_pt: 0.4092 | Regression_sin_phi: 0.0558 | LR: 5.74e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6183s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0377 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.2252 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.0 | uss_MB: 1457.8 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:25:52,999] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 05:25:52,999] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:42,759] INFO : Step 28000/500000 rank0 | Train Loss: 4.3730 | Train Loss Unweighted: 1.6812 | Train Loss Components: Classification: 0.0246 | Classification_binary: 1.3020 | Regression_cos_phi: 0.0039 | Regression_energy: 0.1750 | Regression_eta: 0.0063 | Regression_pt: 0.1660 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.3046 | Classification_binary: 1.3020 | Regression_cos_phi: 0.0356 | Regression_energy: 1.0665 | Regression_eta: 0.0409 | Regression_pt: 1.5923 | Regression_sin_phi: 0.0313 | LR: 6.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6479s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0378 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.2523 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.2 | uss_MB: 1458.0 | child_rss_MB: 22490.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:26:42,761] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:26:42,828] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:26:53,461] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 05:26:53,462] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:53,295] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 05:27:53,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:29,906] INFO : Step 29000/500000 rank0 | Train Loss: 2.3676 | Train Loss Unweighted: 0.9616 | Train Loss Components: Classification: 0.0087 | Classification_binary: 0.7631 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0896 | Regression_eta: 0.0030 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1077 | Classification_binary: 0.7631 | Regression_cos_phi: 0.0182 | Regression_energy: 0.5458 | Regression_eta: 0.0192 | Regression_pt: 0.8973 | Regression_sin_phi: 0.0163 | LR: 6.29e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6315s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0659 | max_tensorboard_logging: 0.2466 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.7 | uss_MB: 1458.5 | child_rss_MB: 22491.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:28:53,907] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 05:28:53,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:29:54,651] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 05:29:54,651] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:17,999] INFO : Step 30000/500000 rank0 | Train Loss: 1.5656 | Train Loss Unweighted: 0.7949 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.6834 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0432 | Regression_eta: 0.0111 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0095 | Classification_binary: 0.6834 | Regression_cos_phi: 0.0447 | Regression_energy: 0.2630 | Regression_eta: 0.0719 | Regression_pt: 0.4342 | Regression_sin_phi: 0.0589 | LR: 6.58e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5182s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1294 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.9 | uss_MB: 1454.8 | child_rss_MB: 22490.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:30:18,000] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:30:18,069] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:30:18,070] INFO : Running validation on rank0 +rank0 [2026-11-14 05:30:18,071] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:30:18,071] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:30:18,191] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 05:30:18,300] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:30:18,402] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:30:18,469] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:30:18,535] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:30:18,601] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:30:18,668] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:30:18,735] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:30:18,841] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:30:18,944] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:30:19,048] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:30:19,116] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:30:19,184] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:30:19,249] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:30:19,313] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:30:19,378] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:30:19,478] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:30:19,485] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:30:19,487] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:30:19,487] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:30:19,491] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:30:19,493] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:30:19,493] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:30:19,497] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:30:19,499] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:30:19,501] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:30:19,502] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:30:19,504] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:30:19,506] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:30:19,508] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:30:19,510] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:30:19,512] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:30:20,939] INFO : VALIDATION | Step=30000/500000 | Train Loss=1.5656 | Valid Loss=5.5991 | Train Loss Unweighted=0.7949 | Valid Loss Unweighted=3.0657 | Stale=1 | ETA=843.4m +rank0 [2026-11-14 05:30:20,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:30:21,014] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:30:21,021] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:30:21,081] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:30:21,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:30:21,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:30:22,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:30:22,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:30:22,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:30:22,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:30:23,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:30:23,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:30:23,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:30:23,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:30:23,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:30:24,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:30:24,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:30:24,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:30:24,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:30:25,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:30:25,239] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:30:25,245] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:30:25,782] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:30:26,174] INFO : Plotted number of elements +rank0 [2026-11-14 05:30:27,377] INFO : Plotted jets +rank0 [2026-11-14 05:30:28,327] INFO : Plotted jet ratio +rank0 [2026-11-14 05:31:05,983] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 05:31:05,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:05,944] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 05:32:05,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:15,912] INFO : Step 31000/500000 rank0 | Train Loss: 3.2095 | Train Loss Unweighted: 1.3787 | Train Loss Components: Classification: 0.0080 | Classification_binary: 1.1178 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1176 | Regression_eta: 0.0056 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0988 | Classification_binary: 1.1178 | Regression_cos_phi: 0.0500 | Regression_energy: 0.7167 | Regression_eta: 0.0365 | Regression_pt: 1.1525 | Regression_sin_phi: 0.0373 | LR: 6.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5216s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0026 | max_backward: 0.0377 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1280 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2778.4 | uss_MB: 1567.3 | child_rss_MB: 22490.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:33:05,885] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 05:33:05,885] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:34:02,199] INFO : Step 32000/500000 rank0 | Train Loss: 2.2315 | Train Loss Unweighted: 1.0825 | Train Loss Components: Classification: 0.0080 | Classification_binary: 0.9182 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0759 | Regression_eta: 0.0057 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0996 | Classification_binary: 0.9182 | Regression_cos_phi: 0.0286 | Regression_energy: 0.4623 | Regression_eta: 0.0368 | Regression_pt: 0.6653 | Regression_sin_phi: 0.0207 | LR: 7.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5284s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0379 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1340 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.4 | uss_MB: 1567.3 | child_rss_MB: 22490.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:34:02,201] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:34:02,271] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:34:05,863] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 05:34:05,863] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:05,749] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 05:35:05,749] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:48,998] INFO : Step 33000/500000 rank0 | Train Loss: 3.5240 | Train Loss Unweighted: 1.9010 | Train Loss Components: Classification: 0.0051 | Classification_binary: 1.6693 | Regression_cos_phi: 0.0040 | Regression_energy: 0.1038 | Regression_eta: 0.0044 | Regression_pt: 0.1037 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 1.6693 | Regression_cos_phi: 0.0366 | Regression_energy: 0.6322 | Regression_eta: 0.0286 | Regression_pt: 0.9943 | Regression_sin_phi: 0.1003 | LR: 7.51e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5224s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2690.4 | uss_MB: 1479.3 | child_rss_MB: 22490.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:36:05,821] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:36:05,821] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:05,626] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:37:05,626] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:35,818] INFO : Step 34000/500000 rank0 | Train Loss: 2.6818 | Train Loss Unweighted: 2.0403 | Train Loss Components: Classification: 0.0044 | Classification_binary: 1.9510 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0344 | Regression_eta: 0.0042 | Regression_pt: 0.0342 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 1.9510 | Regression_cos_phi: 0.0270 | Regression_energy: 0.2095 | Regression_eta: 0.0275 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0836 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5420s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0891 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2690.9 | uss_MB: 1479.8 | child_rss_MB: 22491.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:37:35,820] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:37:35,889] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:38:05,820] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:38:05,820] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:05,494] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:39:05,494] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:22,868] INFO : Step 35000/500000 rank0 | Train Loss: 1.2172 | Train Loss Unweighted: 0.4525 | Train Loss Components: Classification: 0.0052 | Classification_binary: 0.3462 | Regression_cos_phi: 0.0092 | Regression_energy: 0.0374 | Regression_eta: 0.0081 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.3462 | Regression_cos_phi: 0.0836 | Regression_energy: 0.2280 | Regression_eta: 0.0528 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0569 | LR: 8.16e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5366s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1492 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2693.9 | uss_MB: 1482.8 | child_rss_MB: 22491.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:40:06,138] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:40:06,139] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:06,189] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:41:06,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:10,017] INFO : Step 36000/500000 rank0 | Train Loss: 1.4414 | Train Loss Unweighted: 0.9418 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.8706 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0265 | Regression_eta: 0.0064 | Regression_pt: 0.0287 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0170 | Classification_binary: 0.8706 | Regression_cos_phi: 0.0492 | Regression_energy: 0.1617 | Regression_eta: 0.0415 | Regression_pt: 0.2748 | Regression_sin_phi: 0.0265 | LR: 8.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2694.0 | uss_MB: 1482.8 | child_rss_MB: 22490.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:41:10,019] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:41:10,091] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:42:06,522] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:42:06,522] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:42:57,036] INFO : Step 37000/500000 rank0 | Train Loss: 1.4451 | Train Loss Unweighted: 0.7644 | Train Loss Components: Classification: 0.0048 | Classification_binary: 0.6716 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0309 | Regression_eta: 0.0056 | Regression_pt: 0.0387 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.6716 | Regression_cos_phi: 0.0610 | Regression_energy: 0.1884 | Regression_eta: 0.0363 | Regression_pt: 0.3712 | Regression_sin_phi: 0.0567 | LR: 8.85e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5285s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0774 | max_tensorboard_logging: 0.1256 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2694.0 | uss_MB: 1482.9 | child_rss_MB: 22492.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:43:06,838] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:43:06,838] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:07,489] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:44:07,489] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:44,767] INFO : Step 38000/500000 rank0 | Train Loss: 1.5358 | Train Loss Unweighted: 0.9550 | Train Loss Components: Classification: 0.0047 | Classification_binary: 0.8724 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0322 | Regression_eta: 0.0088 | Regression_pt: 0.0303 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.8724 | Regression_cos_phi: 0.0254 | Regression_energy: 0.1963 | Regression_eta: 0.0570 | Regression_pt: 0.2905 | Regression_sin_phi: 0.0358 | LR: 9.21e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5137s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0359 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2691.0 | uss_MB: 1479.8 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:44:44,769] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:44:44,841] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:45:07,850] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:45:07,850] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:07,978] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:46:07,978] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:32,109] INFO : Step 39000/500000 rank0 | Train Loss: 2.1355 | Train Loss Unweighted: 1.2481 | Train Loss Components: Classification: 0.0032 | Classification_binary: 1.1204 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0528 | Regression_eta: 0.0093 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 1.1204 | Regression_cos_phi: 0.0356 | Regression_energy: 0.3215 | Regression_eta: 0.0602 | Regression_pt: 0.4518 | Regression_sin_phi: 0.1068 | LR: 9.57e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5244s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0378 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2693.8 | uss_MB: 1482.6 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:47:08,414] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:47:08,414] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:08,628] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:48:08,629] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:19,499] INFO : Step 40000/500000 rank0 | Train Loss: 1.8948 | Train Loss Unweighted: 0.5663 | Train Loss Components: Classification: 0.0024 | Classification_binary: 0.3738 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0851 | Regression_eta: 0.0097 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0295 | Classification_binary: 0.3738 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5186 | Regression_eta: 0.0627 | Regression_pt: 0.8113 | Regression_sin_phi: 0.0539 | LR: 9.94e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5267s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0360 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1312 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2694.1 | uss_MB: 1483.0 | child_rss_MB: 22489.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:48:19,501] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:48:19,569] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:48:19,570] INFO : Running validation on rank0 +rank0 [2026-11-14 05:48:19,570] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:48:19,570] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:48:19,688] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:48:19,794] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:48:19,896] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:48:19,962] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:48:20,029] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:48:20,096] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:48:20,164] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:48:20,231] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:48:20,341] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:48:20,444] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:48:20,544] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:48:20,610] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:48:20,674] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:48:20,738] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:48:20,802] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:48:20,866] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:48:20,963] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:48:20,969] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:48:20,971] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:48:20,973] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:48:20,975] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:48:20,977] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:48:20,979] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:48:20,981] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:48:20,983] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:48:20,985] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:48:20,987] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:48:20,989] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:48:20,989] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:48:20,991] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:48:20,993] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:48:20,995] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:48:22,396] INFO : VALIDATION | Step=40000/500000 | Train Loss=1.8948 | Valid Loss=7.6595 | Train Loss Unweighted=0.5663 | Valid Loss Unweighted=4.6689 | Stale=2 | ETA=826.3m +rank0 [2026-11-14 05:48:22,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:48:22,837] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:48:22,845] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:48:22,909] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:48:23,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:48:23,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:48:23,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:48:24,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:48:24,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:48:24,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:48:24,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:48:25,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:48:25,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:48:25,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:48:25,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:48:26,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:48:26,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:48:26,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:48:26,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:48:27,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:48:27,118] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:48:27,124] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:48:27,646] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:48:28,022] INFO : Plotted number of elements +rank0 [2026-11-14 05:48:29,270] INFO : Plotted jets +rank0 [2026-11-14 05:48:30,206] INFO : Plotted jet ratio +rank0 [2026-11-14 05:49:20,197] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:49:20,197] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:17,912] INFO : Step 41000/500000 rank0 | Train Loss: 1.2877 | Train Loss Unweighted: 0.4909 | Train Loss Components: Classification: 0.0039 | Classification_binary: 0.3772 | Regression_cos_phi: 0.0025 | Regression_energy: 0.0471 | Regression_eta: 0.0073 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.3772 | Regression_cos_phi: 0.0227 | Regression_energy: 0.2873 | Regression_eta: 0.0473 | Regression_pt: 0.4559 | Regression_sin_phi: 0.0490 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5279s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0028 | max_backward: 0.0386 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1362 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2715.2 | uss_MB: 1504.1 | child_rss_MB: 22490.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 05:50:20,673] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:50:20,673] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:20,503] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:51:20,504] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:04,831] INFO : Step 42000/500000 rank0 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7732 | Train Loss Components: Classification: 0.0011 | Classification_binary: 0.6654 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0501 | Regression_eta: 0.0192 | Regression_pt: 0.0333 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0134 | Classification_binary: 0.6654 | Regression_cos_phi: 0.0182 | Regression_energy: 0.3050 | Regression_eta: 0.1244 | Regression_pt: 0.3190 | Regression_sin_phi: 0.0201 | LR: 1.07e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5187s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1214 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2715.7 | uss_MB: 1504.6 | child_rss_MB: 22492.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:52:04,832] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:52:04,903] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:52:20,859] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:52:20,859] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:53:20,892] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:53:20,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:53:52,037] INFO : Step 43000/500000 rank0 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5793 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.5048 | Regression_cos_phi: 0.0062 | Regression_energy: 0.0290 | Regression_eta: 0.0043 | Regression_pt: 0.0303 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0219 | Classification_binary: 0.5048 | Regression_cos_phi: 0.0561 | Regression_energy: 0.1766 | Regression_eta: 0.0278 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0277 | LR: 1.11e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5116s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1216 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2717.6 | uss_MB: 1506.5 | child_rss_MB: 22492.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:54:21,218] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:54:21,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:55:21,165] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:55:21,165] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:55:38,917] INFO : Step 44000/500000 rank0 | Train Loss: 1.3305 | Train Loss Unweighted: 0.7908 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.7123 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0319 | Regression_eta: 0.0068 | Regression_pt: 0.0341 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0008 | Classification_binary: 0.7123 | Regression_cos_phi: 0.0283 | Regression_energy: 0.1944 | Regression_eta: 0.0439 | Regression_pt: 0.3275 | Regression_sin_phi: 0.0233 | LR: 1.15e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5133s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0030 | max_backward: 0.0391 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1276 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2718.3 | uss_MB: 1507.2 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:55:38,919] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:55:38,987] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:56:21,085] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:56:21,085] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:20,919] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:57:20,919] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:25,647] INFO : Step 45000/500000 rank0 | Train Loss: 2.0639 | Train Loss Unweighted: 1.1701 | Train Loss Components: Classification: 0.0025 | Classification_binary: 1.0413 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0559 | Regression_eta: 0.0062 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0305 | Classification_binary: 1.0413 | Regression_cos_phi: 0.0495 | Regression_energy: 0.3409 | Regression_eta: 0.0403 | Regression_pt: 0.5156 | Regression_sin_phi: 0.0457 | LR: 1.19e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5443s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.1286 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2718.3 | uss_MB: 1507.2 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:58:21,018] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:58:21,018] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:59:12,296] INFO : Step 46000/500000 rank0 | Train Loss: 1.7060 | Train Loss Unweighted: 1.1763 | Train Loss Components: Classification: 0.0026 | Classification_binary: 1.1005 | Regression_cos_phi: 0.0033 | Regression_energy: 0.0300 | Regression_eta: 0.0065 | Regression_pt: 0.0274 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 1.1005 | Regression_cos_phi: 0.0300 | Regression_energy: 0.1830 | Regression_eta: 0.0423 | Regression_pt: 0.2628 | Regression_sin_phi: 0.0548 | LR: 1.23e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5139s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0709 | max_tensorboard_logging: 0.1206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2715.7 | uss_MB: 1504.7 | child_rss_MB: 22490.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:59:12,298] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 05:59:12,369] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:59:21,265] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 05:59:21,265] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:21,050] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 06:00:21,050] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:59,191] INFO : Step 47000/500000 rank0 | Train Loss: 1.4635 | Train Loss Unweighted: 0.9193 | Train Loss Components: Classification: 0.0075 | Classification_binary: 0.8462 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0254 | Regression_eta: 0.0042 | Regression_pt: 0.0299 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0931 | Classification_binary: 0.8462 | Regression_cos_phi: 0.0399 | Regression_energy: 0.1548 | Regression_eta: 0.0274 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0158 | LR: 1.27e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5281s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0378 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1343 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2717.7 | uss_MB: 1506.6 | child_rss_MB: 22491.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 06:01:21,241] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 06:01:21,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:21,523] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 06:02:21,523] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:46,325] INFO : Step 48000/500000 rank0 | Train Loss: 1.6065 | Train Loss Unweighted: 0.7754 | Train Loss Components: Classification: 0.0053 | Classification_binary: 0.6572 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0501 | Regression_eta: 0.0066 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.6572 | Regression_cos_phi: 0.0607 | Regression_energy: 0.3050 | Regression_eta: 0.0425 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0229 | LR: 1.31e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5355s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0441 | max_optimizer_step: 0.0740 | max_tensorboard_logging: 0.1294 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2718.0 | uss_MB: 1506.9 | child_rss_MB: 22489.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 06:02:46,327] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:02:46,395] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:03:21,554] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 06:03:21,554] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:21,629] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 06:04:21,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:33,363] INFO : Step 49000/500000 rank0 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4409 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.3646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0363 | Regression_eta: 0.0029 | Regression_pt: 0.0331 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0008 | Classification_binary: 0.3646 | Regression_cos_phi: 0.0176 | Regression_energy: 0.2214 | Regression_eta: 0.0186 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0189 | LR: 1.36e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5291s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1214 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2718.0 | uss_MB: 1506.9 | child_rss_MB: 22489.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 06:05:22,059] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 06:05:22,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:20,967] INFO : Step 50000/500000 rank0 | Train Loss: 1.1965 | Train Loss Unweighted: 0.5216 | Train Loss Components: Classification: 0.0022 | Classification_binary: 0.4239 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0450 | Regression_eta: 0.0039 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0277 | Classification_binary: 0.4239 | Regression_cos_phi: 0.0181 | Regression_energy: 0.2740 | Regression_eta: 0.0251 | Regression_pt: 0.4062 | Regression_sin_phi: 0.0215 | LR: 1.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5256s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2715.4 | uss_MB: 1504.3 | child_rss_MB: 22491.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 06:06:20,969] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:06:21,039] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:06:21,040] INFO : Running validation on rank0 +rank0 [2026-11-14 06:06:21,040] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:06:21,040] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:06:21,170] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 06:06:21,278] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:06:21,380] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:06:21,449] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:06:21,514] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:06:21,579] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:06:21,645] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:06:21,710] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:06:21,818] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:06:21,918] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:06:22,016] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:06:22,084] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:06:22,149] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:06:22,213] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:06:22,278] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:06:22,342] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:06:22,441] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:06:22,447] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:06:22,448] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:06:22,451] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:06:22,453] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:06:22,455] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:06:22,457] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:06:22,459] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:06:22,461] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:06:22,463] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:06:22,465] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:06:22,466] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:06:22,468] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:06:22,469] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:06:22,471] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:06:22,474] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:06:23,887] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.1965 | Valid Loss=9.5837 | Train Loss Unweighted=0.5216 | Valid Loss Unweighted=6.3989 | Stale=3 | ETA=808.9m +rank0 [2026-11-14 06:06:23,888] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:06:23,958] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 06:06:23,967] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 06:06:24,033] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:06:24,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 06:06:24,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 06:06:25,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 06:06:25,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 06:06:25,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 06:06:25,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 06:06:26,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 06:06:26,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 06:06:26,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 06:06:26,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 06:06:27,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 06:06:27,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 06:06:27,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 06:06:27,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 06:06:27,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 06:06:28,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 06:06:28,288] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:06:28,293] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 06:06:28,822] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 06:06:29,196] INFO : Plotted number of elements +rank0 [2026-11-14 06:06:30,426] INFO : Plotted jets +rank0 [2026-11-14 06:06:31,349] INFO : Plotted jet ratio +rank0 [2026-11-14 06:06:31,768] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 06:06:31,768] INFO : Training completed. Total time on device 0: 90.012min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..064aa2dc7a20109a65c21371a5acffe0dfcf3440 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/train.log.0 @@ -0,0 +1,997 @@ +rank0 [2026-11-14 04:36:28,427] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 04:36:28,428] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 04:36:28,428] INFO : Instantiating model +rank0 [2026-11-14 04:36:28,450] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 04:36:28,450] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 04:36:28,452] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 04:36:28,452] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 04:36:28,452] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 04:36:28,452] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 04:36:28,452] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,455] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 04:36:28,455] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,458] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 04:36:28,458] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,460] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 04:36:28,461] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,463] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 04:36:28,463] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,466] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 04:36:28,466] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,469] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 04:36:28,469] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 04:36:28,469] INFO : Initializing output DNNs +rank0 [2026-11-14 04:36:28,470] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,473] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 04:36:28,482] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 04:36:28,482] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 04:36:28,482] INFO : jagged_attention enabled=True +rank0 [2026-11-14 04:36:28,482] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 04:36:28,482] INFO : task_queries enabled=True +rank0 [2026-11-14 04:36:28,482] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 04:36:28,482] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 04:36:28,482] INFO : classification_query parameters: 256 +rank0 [2026-11-14 04:36:28,482] INFO : regression_query parameters: 256 +rank0 [2026-11-14 04:36:28,482] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 04:36:28,482] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 04:36:28,482] INFO : output_mode=elementwise +rank0 [2026-11-14 04:36:28,482] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 04:36:28,482] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 04:36:28,482] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 04:36:28,482] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 04:36:28,483] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 04:36:28,483] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 04:36:28,483] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 04:36:28,483] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 04:36:28,484] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 04:36:28,484] INFO : MLPF __init__ done +rank0 [2026-11-14 04:36:28,484] INFO : Instantiated model +rank0 [2026-11-14 04:36:28,485] INFO : Moving model to device rank=0 +rank0 [2026-11-14 04:36:28,495] INFO : Moved model to device rank=0 +rank0 [2026-11-14 04:36:28,496] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 04:36:28,592] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 04:36:28,592] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 04:36:28,594] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 04:36:28,594] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 04:36:28,594] INFO : Total parameters: 3910930 +rank0 [2026-11-14 04:36:28,596] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 04:36:28,599] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596 +rank0 [2026-11-14 04:36:28,599] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596 +rank0 [2026-11-14 04:36:28,599] INFO : Getting dataloaders +rank0 [2026-11-14 04:36:28,599] INFO : sample=cld_edm_z_tautau_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:36:29,927] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:29,969] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:36:30,009] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:36:30,064] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,121] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,532] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,587] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,639] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,684] INFO : train_dataset: cld_edm_z_tautau_pf, 899 +rank0 [2026-11-14 04:36:30,722] INFO : train_dataset: cld_edm_z_tautau_pf, 900 +rank0 [2026-11-14 04:36:30,722] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 04:36:30,722] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 04:36:30,722] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 04:36:30,722] INFO : sample=cld_edm_z_tautau_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:36:30,734] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,747] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,757] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,768] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,779] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,792] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,804] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,815] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,827] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,840] INFO : valid_dataset: cld_edm_z_tautau_pf, 25 +rank0 [2026-11-14 04:36:30,840] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 04:36:30,840] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 04:36:30,888] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:36:31,034] INFO : Got dataloaders +rank0 [2026-11-14 04:36:31,034] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 04:36:31,034] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 04:36:31,035] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 04:36:31,040] INFO : Creating train_iterator +rank0 [2026-11-14 04:36:31,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:36:31,071] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:36:31,290] INFO : Created train_iterator +rank0 [2026-11-14 04:36:43,204] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.40914969921112, 'Classification': 0.5980110427737236, 'Regression_pt': 0.7724759268760681, 'Regression_eta': 1.1418460792303085, 'Regression_sin_phi': 0.7991263212263584, 'Regression_cos_phi': 0.813458954244852, 'Regression_energy': 1.2159733620285988}, weights={'Classification_binary': 1.0, 'Classification': 12.389653205871582, 'Regression_pt': 9.591431617736816, 'Regression_eta': 6.488746643066406, 'Regression_sin_phi': 9.271562576293945, 'Regression_cos_phi': 9.108203887939453, 'Regression_energy': 6.093183994293213} +rank0 [2026-11-14 04:37:32,778] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 04:37:32,778] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:38:19,749] INFO : Step 1000/500000 rank0 | Train Loss: 8.8077 | Train Loss Unweighted: 4.5601 | Train Loss Components: Classification: 0.0803 | Classification_binary: 4.0058 | Regression_cos_phi: 0.0553 | Regression_energy: 0.1631 | Regression_eta: 0.0423 | Regression_pt: 0.1777 | Regression_sin_phi: 0.0355 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.9954 | Classification_binary: 4.0058 | Regression_cos_phi: 0.5041 | Regression_energy: 0.9940 | Regression_eta: 0.2747 | Regression_pt: 1.7042 | Regression_sin_phi: 0.3296 | LR: 2.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6007s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0002 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0648 | max_tensorboard_logging: 0.2241 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2559.2 | uss_MB: 1340.5 | child_rss_MB: 22496.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:38:32,874] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 04:38:32,874] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:39:32,297] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 04:39:32,297] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:40:06,202] INFO : Step 2000/500000 rank0 | Train Loss: 8.7654 | Train Loss Unweighted: 3.8513 | Train Loss Components: Classification: 0.0585 | Classification_binary: 3.1757 | Regression_cos_phi: 0.0170 | Regression_energy: 0.2721 | Regression_eta: 0.0290 | Regression_pt: 0.2848 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.7251 | Classification_binary: 3.1757 | Regression_cos_phi: 0.1545 | Regression_energy: 1.6577 | Regression_eta: 0.1880 | Regression_pt: 2.7316 | Regression_sin_phi: 0.1328 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5730s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0027 | max_backward: 0.0383 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1994 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2559.9 | uss_MB: 1341.3 | child_rss_MB: 22489.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:40:06,204] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:40:32,627] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 04:40:32,627] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:32,678] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 04:41:32,678] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:41:53,419] INFO : Step 3000/500000 rank0 | Train Loss: 7.0901 | Train Loss Unweighted: 3.3502 | Train Loss Components: Classification: 0.0940 | Classification_binary: 2.8941 | Regression_cos_phi: 0.1078 | Regression_energy: 0.0792 | Regression_eta: 0.0279 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0815 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 1.1640 | Classification_binary: 2.8941 | Regression_cos_phi: 0.9818 | Regression_energy: 0.4827 | Regression_eta: 0.1809 | Regression_pt: 0.6308 | Regression_sin_phi: 0.7557 | LR: 2.05e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5709s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1973 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2563.2 | uss_MB: 1344.9 | child_rss_MB: 22489.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:42:32,945] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 04:42:32,945] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:32,683] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 04:43:32,683] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:43:40,178] INFO : Step 4000/500000 rank0 | Train Loss: 6.7464 | Train Loss Unweighted: 3.9315 | Train Loss Components: Classification: 0.0468 | Classification_binary: 3.5623 | Regression_cos_phi: 0.0173 | Regression_energy: 0.1228 | Regression_eta: 0.0136 | Regression_pt: 0.1434 | Regression_sin_phi: 0.0253 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.5801 | Classification_binary: 3.5623 | Regression_cos_phi: 0.1578 | Regression_energy: 0.7483 | Regression_eta: 0.0883 | Regression_pt: 1.3750 | Regression_sin_phi: 0.2346 | LR: 2.08e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5974s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0339 | max_loss: 0.0024 | max_backward: 0.0378 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.2266 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2563.2 | uss_MB: 1344.9 | child_rss_MB: 22489.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:43:40,180] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:44:32,761] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 04:44:32,762] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:45:27,192] INFO : Step 5000/500000 rank0 | Train Loss: 7.9320 | Train Loss Unweighted: 3.2771 | Train Loss Components: Classification: 0.0128 | Classification_binary: 2.6180 | Regression_cos_phi: 0.0064 | Regression_energy: 0.2855 | Regression_eta: 0.0127 | Regression_pt: 0.3339 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1590 | Classification_binary: 2.6180 | Regression_cos_phi: 0.0581 | Regression_energy: 1.7395 | Regression_eta: 0.0823 | Regression_pt: 3.2024 | Regression_sin_phi: 0.0727 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5407s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0387 | max_optimizer_step: 0.0652 | max_tensorboard_logging: 0.1752 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2561.1 | uss_MB: 1342.8 | child_rss_MB: 22489.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 04:45:33,376] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 04:45:33,376] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:46:32,760] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 04:46:32,760] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:47:13,246] INFO : Step 6000/500000 rank0 | Train Loss: 8.4547 | Train Loss Unweighted: 3.5909 | Train Loss Components: Classification: 0.0100 | Classification_binary: 2.8883 | Regression_cos_phi: 0.0062 | Regression_energy: 0.3349 | Regression_eta: 0.0077 | Regression_pt: 0.3378 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1236 | Classification_binary: 2.8883 | Regression_cos_phi: 0.0563 | Regression_energy: 2.0404 | Regression_eta: 0.0497 | Regression_pt: 3.2403 | Regression_sin_phi: 0.0560 | LR: 2.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0341 | max_loss: 0.0026 | max_backward: 0.0419 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1976 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2561.1 | uss_MB: 1342.8 | child_rss_MB: 22489.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:47:13,248] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:47:32,474] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 04:47:32,474] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:31,772] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 04:48:31,773] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:48:59,500] INFO : Step 7000/500000 rank0 | Train Loss: 2.6730 | Train Loss Unweighted: 1.7274 | Train Loss Components: Classification: 0.0235 | Classification_binary: 1.6060 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0344 | Regression_eta: 0.0120 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2908 | Classification_binary: 1.6060 | Regression_cos_phi: 0.0666 | Regression_energy: 0.2094 | Regression_eta: 0.0779 | Regression_pt: 0.3730 | Regression_sin_phi: 0.0494 | LR: 2.26e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5776s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0659 | max_tensorboard_logging: 0.2030 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2561.9 | uss_MB: 1343.6 | child_rss_MB: 22488.9 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:49:32,121] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 04:49:32,121] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:31,481] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 04:50:31,481] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:50:46,044] INFO : Step 8000/500000 rank0 | Train Loss: 3.5124 | Train Loss Unweighted: 1.6850 | Train Loss Components: Classification: 0.0275 | Classification_binary: 1.4560 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0332 | Regression_eta: 0.0163 | Regression_pt: 0.0293 | Regression_sin_phi: 0.0569 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.3403 | Classification_binary: 1.4560 | Regression_cos_phi: 0.5998 | Regression_energy: 0.2023 | Regression_eta: 0.1057 | Regression_pt: 0.2810 | Regression_sin_phi: 0.5274 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5761s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0376 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.2107 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2561.9 | uss_MB: 1343.7 | child_rss_MB: 22488.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:50:46,046] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 04:50:46,117] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 04:51:31,865] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 04:51:31,865] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:31,589] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 04:52:31,589] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:52:32,930] INFO : Step 9000/500000 rank0 | Train Loss: 5.5896 | Train Loss Unweighted: 3.0148 | Train Loss Components: Classification: 0.0050 | Classification_binary: 2.6483 | Regression_cos_phi: 0.0154 | Regression_energy: 0.1529 | Regression_eta: 0.0139 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 2.6483 | Regression_cos_phi: 0.1403 | Regression_energy: 0.9317 | Regression_eta: 0.0899 | Regression_pt: 1.6651 | Regression_sin_phi: 0.0519 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5686s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0379 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.2062 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2560.1 | uss_MB: 1341.9 | child_rss_MB: 22490.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:53:32,096] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 04:53:32,099] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:54:20,112] INFO : Step 10000/500000 rank0 | Train Loss: 5.0393 | Train Loss Unweighted: 2.6469 | Train Loss Components: Classification: 0.0124 | Classification_binary: 2.3063 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1545 | Regression_eta: 0.0080 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1538 | Classification_binary: 2.3063 | Regression_cos_phi: 0.0460 | Regression_energy: 0.9412 | Regression_eta: 0.0521 | Regression_pt: 1.4942 | Regression_sin_phi: 0.0457 | LR: 2.52e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6434s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0375 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.2650 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2560.7 | uss_MB: 1342.4 | child_rss_MB: 22490.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:54:20,114] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 04:54:20,185] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 04:54:20,186] INFO : Running validation on rank0 +rank0 [2026-11-14 04:54:20,186] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 04:54:20,186] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 04:54:20,351] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 04:54:20,525] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 04:54:20,592] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 04:54:20,644] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 04:54:20,699] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 04:54:20,744] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 04:54:20,797] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 04:54:20,850] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 04:54:20,897] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 04:54:20,961] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 04:54:21,026] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 04:54:21,079] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 04:54:21,134] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 04:54:21,178] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 04:54:21,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 04:54:21,285] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 04:54:21,334] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 04:54:21,337] INFO : plotted pt_target histogram +rank0 [2026-11-14 04:54:21,340] INFO : plotted pt_pred histogram +rank0 [2026-11-14 04:54:21,342] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 04:54:21,343] INFO : plotted eta_target histogram +rank0 [2026-11-14 04:54:21,346] INFO : plotted eta_pred histogram +rank0 [2026-11-14 04:54:21,348] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 04:54:21,350] INFO : plotted sphi_target histogram +rank0 [2026-11-14 04:54:21,352] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 04:54:21,353] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 04:54:21,355] INFO : plotted cphi_target histogram +rank0 [2026-11-14 04:54:21,356] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 04:54:21,358] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 04:54:21,360] INFO : plotted energy_target histogram +rank0 [2026-11-14 04:54:21,362] INFO : plotted energy_pred histogram +rank0 [2026-11-14 04:54:21,364] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 04:54:22,833] INFO : VALIDATION | Step=10000/500000 | Train Loss=5.0393 | Valid Loss=6.0359 | Train Loss Unweighted=2.6469 | Valid Loss Unweighted=2.9008 | Stale=0 | ETA=875.3m +rank0 [2026-11-14 04:54:22,834] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 04:54:22,948] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 04:54:22,956] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:23,018] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 04:54:23,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 04:54:23,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 04:54:24,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 04:54:24,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 04:54:24,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 04:54:24,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 04:54:25,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 04:54:25,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 04:54:25,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 04:54:25,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 04:54:26,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 04:54:26,348] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 04:54:26,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 04:54:26,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 04:54:27,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 04:54:27,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 04:54:27,350] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 04:54:27,358] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:27,883] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_10000/cld_edm_z_tautau_pf +rank0 [2026-11-14 04:54:28,588] INFO : Plotted number of elements +rank0 [2026-11-14 04:54:29,859] INFO : Plotted jets +rank0 [2026-11-14 04:54:30,792] INFO : Plotted jet ratio +rank0 [2026-11-14 04:54:43,496] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 04:54:43,496] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:55:43,259] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 04:55:43,259] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:56:17,954] INFO : Step 11000/500000 rank0 | Train Loss: 4.7022 | Train Loss Unweighted: 3.0012 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.7658 | Regression_cos_phi: 0.0052 | Regression_energy: 0.0974 | Regression_eta: 0.0099 | Regression_pt: 0.0958 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2399 | Classification_binary: 2.7658 | Regression_cos_phi: 0.0472 | Regression_energy: 0.5933 | Regression_eta: 0.0640 | Regression_pt: 0.9188 | Regression_sin_phi: 0.0734 | LR: 2.63e-05 | DataLoad Time: 0.0028s | Model Forward Time: 0.6073s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0410 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.2278 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2683.5 | uss_MB: 1472.3 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 04:56:43,394] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 04:56:43,394] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:57:42,672] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 04:57:42,672] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:58:04,117] INFO : Step 12000/500000 rank0 | Train Loss: 4.5926 | Train Loss Unweighted: 2.1270 | Train Loss Components: Classification: 0.0121 | Classification_binary: 1.7757 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1579 | Regression_eta: 0.0097 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1504 | Classification_binary: 1.7757 | Regression_cos_phi: 0.0413 | Regression_energy: 0.9618 | Regression_eta: 0.0628 | Regression_pt: 1.5394 | Regression_sin_phi: 0.0612 | LR: 2.75e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5706s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0340 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1847 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2683.5 | uss_MB: 1472.3 | child_rss_MB: 22490.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 04:58:04,119] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 04:58:04,191] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 04:58:42,542] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 04:58:42,542] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:42,219] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 04:59:42,219] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 04:59:50,600] INFO : Step 13000/500000 rank0 | Train Loss: 3.7013 | Train Loss Unweighted: 1.9955 | Train Loss Components: Classification: 0.0233 | Classification_binary: 1.7637 | Regression_cos_phi: 0.0099 | Regression_energy: 0.0846 | Regression_eta: 0.0155 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2881 | Classification_binary: 1.7637 | Regression_cos_phi: 0.0906 | Regression_energy: 0.5152 | Regression_eta: 0.1004 | Regression_pt: 0.8802 | Regression_sin_phi: 0.0631 | LR: 2.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6099s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0360 | max_loss: 0.0025 | max_backward: 0.0375 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.2073 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2632.1 | uss_MB: 1420.9 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:00:42,380] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:00:42,380] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:01:37,169] INFO : Step 14000/500000 rank0 | Train Loss: 2.9774 | Train Loss Unweighted: 1.7265 | Train Loss Components: Classification: 0.0048 | Classification_binary: 1.5469 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0825 | Regression_eta: 0.0044 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 1.5469 | Regression_cos_phi: 0.0531 | Regression_energy: 0.5028 | Regression_eta: 0.0285 | Regression_pt: 0.7710 | Regression_sin_phi: 0.0161 | LR: 3.02e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5163s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0377 | max_optimizer_step: 0.0647 | max_tensorboard_logging: 0.1457 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2632.8 | uss_MB: 1421.6 | child_rss_MB: 22490.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:01:37,171] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:01:37,240] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:01:42,540] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 05:01:42,540] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:02:42,256] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 05:02:42,257] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:24,400] INFO : Step 15000/500000 rank0 | Train Loss: 4.5850 | Train Loss Unweighted: 2.9683 | Train Loss Components: Classification: 0.0122 | Classification_binary: 2.7415 | Regression_cos_phi: 0.0062 | Regression_energy: 0.0939 | Regression_eta: 0.0105 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1517 | Classification_binary: 2.7415 | Regression_cos_phi: 0.0565 | Regression_energy: 0.5722 | Regression_eta: 0.0684 | Regression_pt: 0.9436 | Regression_sin_phi: 0.0512 | LR: 3.17e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0391 | max_optimizer_step: 0.0765 | max_tensorboard_logging: 0.1767 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2637.2 | uss_MB: 1426.0 | child_rss_MB: 22490.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:03:42,858] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 05:03:42,858] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:42,412] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 05:04:42,412] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:10,914] INFO : Step 16000/500000 rank0 | Train Loss: 3.9743 | Train Loss Unweighted: 2.3588 | Train Loss Components: Classification: 0.0034 | Classification_binary: 2.1258 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1079 | Regression_eta: 0.0042 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 2.1258 | Regression_cos_phi: 0.0655 | Regression_energy: 0.6575 | Regression_eta: 0.0275 | Regression_pt: 1.0183 | Regression_sin_phi: 0.0369 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.6023s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0374 | max_optimizer_step: 0.0771 | max_tensorboard_logging: 0.2102 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2637.3 | uss_MB: 1426.1 | child_rss_MB: 22489.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:05:10,916] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:05:10,984] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:05:42,622] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 05:05:42,622] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:42,357] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 05:06:42,357] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:57,711] INFO : Step 17000/500000 rank0 | Train Loss: 3.8166 | Train Loss Unweighted: 2.5697 | Train Loss Components: Classification: 0.0080 | Classification_binary: 2.3963 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0670 | Regression_eta: 0.0077 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0146 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0993 | Classification_binary: 2.3963 | Regression_cos_phi: 0.0386 | Regression_energy: 0.4080 | Regression_eta: 0.0497 | Regression_pt: 0.6892 | Regression_sin_phi: 0.1355 | LR: 3.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5717s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0374 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1918 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2632.0 | uss_MB: 1420.8 | child_rss_MB: 22489.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:07:42,892] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 05:07:42,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:42,636] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 05:08:42,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:44,816] INFO : Step 18000/500000 rank0 | Train Loss: 3.6751 | Train Loss Unweighted: 1.5132 | Train Loss Components: Classification: 0.0106 | Classification_binary: 1.2169 | Regression_cos_phi: 0.0044 | Regression_energy: 0.1062 | Regression_eta: 0.0122 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1309 | Classification_binary: 1.2169 | Regression_cos_phi: 0.0398 | Regression_energy: 0.6473 | Regression_eta: 0.0791 | Regression_pt: 1.5062 | Regression_sin_phi: 0.0549 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5743s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0375 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1956 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2632.8 | uss_MB: 1421.6 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:08:44,818] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:08:44,889] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:09:42,890] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 05:09:42,890] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:31,506] INFO : Step 19000/500000 rank0 | Train Loss: 2.9176 | Train Loss Unweighted: 1.6474 | Train Loss Components: Classification: 0.0170 | Classification_binary: 1.4739 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0714 | Regression_eta: 0.0046 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2102 | Classification_binary: 1.4739 | Regression_cos_phi: 0.0387 | Regression_energy: 0.4349 | Regression_eta: 0.0301 | Regression_pt: 0.6964 | Regression_sin_phi: 0.0334 | LR: 3.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5904s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0369 | max_loss: 0.0029 | max_backward: 0.0392 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1839 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2637.0 | uss_MB: 1425.9 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:10:42,909] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 05:10:42,910] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:42,738] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 05:11:42,738] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:18,081] INFO : Step 20000/500000 rank0 | Train Loss: 1.3325 | Train Loss Unweighted: 0.6857 | Train Loss Components: Classification: 0.0087 | Classification_binary: 0.5977 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0348 | Regression_eta: 0.0031 | Regression_pt: 0.0365 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1079 | Classification_binary: 0.5977 | Regression_cos_phi: 0.0246 | Regression_energy: 0.2122 | Regression_eta: 0.0201 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0200 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5670s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0350 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0643 | max_tensorboard_logging: 0.1982 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2637.4 | uss_MB: 1426.2 | child_rss_MB: 22490.2 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:12:18,083] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:12:18,151] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:12:18,152] INFO : Running validation on rank0 +rank0 [2026-11-14 05:12:18,152] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:12:18,153] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:12:18,271] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 05:12:18,377] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:12:18,475] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:12:18,542] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:12:18,606] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:12:18,670] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:12:18,735] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:12:18,801] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:12:18,900] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:12:19,000] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:12:19,101] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:12:19,170] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:12:19,234] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:12:19,299] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:12:19,363] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:12:19,426] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:12:19,533] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:12:19,537] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:12:19,541] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:12:19,543] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:12:19,545] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:12:19,547] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:12:19,549] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:12:19,551] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:12:19,553] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:12:19,555] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:12:19,557] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:12:19,558] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:12:19,559] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:12:19,561] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:12:19,563] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:12:19,564] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:12:21,041] INFO : VALIDATION | Step=20000/500000 | Train Loss=1.3325 | Valid Loss=5.3294 | Train Loss Unweighted=0.6857 | Valid Loss Unweighted=2.6964 | Stale=0 | ETA=860.0m +rank0 [2026-11-14 05:12:21,042] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:12:21,140] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:12:21,147] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:12:21,208] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:12:21,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:12:21,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:12:22,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:12:22,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:12:22,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:12:22,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:12:23,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:12:23,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:12:23,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:12:23,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:12:24,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:12:24,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:12:24,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:12:24,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:12:25,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:12:25,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:12:25,365] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:12:25,372] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:12:25,895] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_20000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:12:26,282] INFO : Plotted number of elements +rank0 [2026-11-14 05:12:27,477] INFO : Plotted jets +rank0 [2026-11-14 05:12:28,407] INFO : Plotted jet ratio +rank0 [2026-11-14 05:12:53,420] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 05:12:53,420] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:52,902] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 05:13:52,902] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:14:15,261] INFO : Step 21000/500000 rank0 | Train Loss: 4.2539 | Train Loss Unweighted: 2.0038 | Train Loss Components: Classification: 0.0094 | Classification_binary: 1.6809 | Regression_cos_phi: 0.0037 | Regression_energy: 0.1524 | Regression_eta: 0.0049 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1161 | Classification_binary: 1.6809 | Regression_cos_phi: 0.0335 | Regression_energy: 0.9283 | Regression_eta: 0.0317 | Regression_pt: 1.4431 | Regression_sin_phi: 0.0203 | LR: 4.28e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6420s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0342 | max_loss: 0.0025 | max_backward: 0.0378 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.2521 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2665.0 | uss_MB: 1453.8 | child_rss_MB: 22491.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:14:52,918] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 05:14:52,918] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:52,314] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 05:15:52,314] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:01,558] INFO : Step 22000/500000 rank0 | Train Loss: 2.9036 | Train Loss Unweighted: 1.2899 | Train Loss Components: Classification: 0.0030 | Classification_binary: 1.0606 | Regression_cos_phi: 0.0082 | Regression_energy: 0.0936 | Regression_eta: 0.0100 | Regression_pt: 0.1082 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 1.0606 | Regression_cos_phi: 0.0751 | Regression_energy: 0.5704 | Regression_eta: 0.0649 | Regression_pt: 1.0375 | Regression_sin_phi: 0.0583 | LR: 4.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6443s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0382 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2435 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2666.0 | uss_MB: 1454.8 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:16:01,560] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:16:01,629] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:16:52,963] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 05:16:52,963] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:48,626] INFO : Step 23000/500000 rank0 | Train Loss: 3.4638 | Train Loss Unweighted: 1.9056 | Train Loss Components: Classification: 0.0197 | Classification_binary: 1.6909 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0931 | Regression_eta: 0.0042 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.2447 | Classification_binary: 1.6909 | Regression_cos_phi: 0.0264 | Regression_energy: 0.5675 | Regression_eta: 0.0275 | Regression_pt: 0.8747 | Regression_sin_phi: 0.0321 | LR: 4.73e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.6491s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0355 | max_loss: 0.0026 | max_backward: 0.0379 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.2637 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.1 | uss_MB: 1457.9 | child_rss_MB: 22490.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:17:53,083] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 05:17:53,083] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:52,422] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 05:18:52,422] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:34,732] INFO : Step 24000/500000 rank0 | Train Loss: 2.2348 | Train Loss Unweighted: 1.4395 | Train Loss Components: Classification: 0.0048 | Classification_binary: 1.3260 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0516 | Regression_eta: 0.0035 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 1.3260 | Regression_cos_phi: 0.0266 | Regression_energy: 0.3146 | Regression_eta: 0.0230 | Regression_pt: 0.4571 | Regression_sin_phi: 0.0276 | LR: 4.97e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5764s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0343 | max_loss: 0.0026 | max_backward: 0.0376 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1896 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.1 | uss_MB: 1457.9 | child_rss_MB: 22490.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:19:34,734] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:19:34,801] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:19:52,328] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 05:19:52,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:51,844] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 05:20:51,844] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:21,408] INFO : Step 25000/500000 rank0 | Train Loss: 7.9381 | Train Loss Unweighted: 2.9835 | Train Loss Components: Classification: 0.0137 | Classification_binary: 2.2697 | Regression_cos_phi: 0.0093 | Regression_energy: 0.3387 | Regression_eta: 0.0078 | Regression_pt: 0.3349 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1698 | Classification_binary: 2.2697 | Regression_cos_phi: 0.0846 | Regression_energy: 2.0636 | Regression_eta: 0.0509 | Regression_pt: 3.2118 | Regression_sin_phi: 0.0878 | LR: 5.22e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5996s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0343 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.1933 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.1 | uss_MB: 1453.9 | child_rss_MB: 22488.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:21:52,293] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 05:21:52,293] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:52,204] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 05:22:52,204] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:23:08,434] INFO : Step 26000/500000 rank0 | Train Loss: 3.2594 | Train Loss Unweighted: 1.0386 | Train Loss Components: Classification: 0.0068 | Classification_binary: 0.7194 | Regression_cos_phi: 0.0033 | Regression_energy: 0.1494 | Regression_eta: 0.0050 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0839 | Classification_binary: 0.7194 | Regression_cos_phi: 0.0298 | Regression_energy: 0.9102 | Regression_eta: 0.0327 | Regression_pt: 1.4380 | Regression_sin_phi: 0.0453 | LR: 5.47e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5989s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.2022 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.9 | uss_MB: 1454.7 | child_rss_MB: 22492.1 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:23:08,436] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:23:08,507] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:23:52,476] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 05:23:52,477] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:52,303] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 05:24:52,303] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:55,384] INFO : Step 27000/500000 rank0 | Train Loss: 2.7915 | Train Loss Unweighted: 1.9927 | Train Loss Components: Classification: 0.0085 | Classification_binary: 1.8815 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0449 | Regression_eta: 0.0067 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1050 | Classification_binary: 1.8815 | Regression_cos_phi: 0.0233 | Regression_energy: 0.2734 | Regression_eta: 0.0433 | Regression_pt: 0.4092 | Regression_sin_phi: 0.0558 | LR: 5.74e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6183s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0377 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.2252 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.0 | uss_MB: 1457.8 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:25:52,999] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 05:25:52,999] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:42,759] INFO : Step 28000/500000 rank0 | Train Loss: 4.3730 | Train Loss Unweighted: 1.6812 | Train Loss Components: Classification: 0.0246 | Classification_binary: 1.3020 | Regression_cos_phi: 0.0039 | Regression_energy: 0.1750 | Regression_eta: 0.0063 | Regression_pt: 0.1660 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.3046 | Classification_binary: 1.3020 | Regression_cos_phi: 0.0356 | Regression_energy: 1.0665 | Regression_eta: 0.0409 | Regression_pt: 1.5923 | Regression_sin_phi: 0.0313 | LR: 6.01e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6479s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0378 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.2523 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.2 | uss_MB: 1458.0 | child_rss_MB: 22490.8 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:26:42,761] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:26:42,828] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:26:53,461] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 05:26:53,462] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:53,295] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 05:27:53,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:29,906] INFO : Step 29000/500000 rank0 | Train Loss: 2.3676 | Train Loss Unweighted: 0.9616 | Train Loss Components: Classification: 0.0087 | Classification_binary: 0.7631 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0896 | Regression_eta: 0.0030 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.1077 | Classification_binary: 0.7631 | Regression_cos_phi: 0.0182 | Regression_energy: 0.5458 | Regression_eta: 0.0192 | Regression_pt: 0.8973 | Regression_sin_phi: 0.0163 | LR: 6.29e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.6315s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0659 | max_tensorboard_logging: 0.2466 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2669.7 | uss_MB: 1458.5 | child_rss_MB: 22491.0 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:28:53,907] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 05:28:53,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:29:54,651] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 05:29:54,651] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:17,999] INFO : Step 30000/500000 rank0 | Train Loss: 1.5656 | Train Loss Unweighted: 0.7949 | Train Loss Components: Classification: 0.0008 | Classification_binary: 0.6834 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0432 | Regression_eta: 0.0111 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0095 | Classification_binary: 0.6834 | Regression_cos_phi: 0.0447 | Regression_energy: 0.2630 | Regression_eta: 0.0719 | Regression_pt: 0.4342 | Regression_sin_phi: 0.0589 | LR: 6.58e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5182s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1294 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.9 | uss_MB: 1454.8 | child_rss_MB: 22490.5 | allocated_MB: 125.2 | reserved_MB: 250.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:30:18,000] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:30:18,069] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:30:18,070] INFO : Running validation on rank0 +rank0 [2026-11-14 05:30:18,071] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:30:18,071] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:30:18,191] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 05:30:18,300] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:30:18,402] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:30:18,469] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:30:18,535] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:30:18,601] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:30:18,668] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:30:18,735] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:30:18,841] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:30:18,944] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:30:19,048] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:30:19,116] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:30:19,184] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:30:19,249] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:30:19,313] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:30:19,378] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:30:19,478] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:30:19,485] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:30:19,487] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:30:19,487] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:30:19,491] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:30:19,493] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:30:19,493] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:30:19,497] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:30:19,499] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:30:19,501] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:30:19,502] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:30:19,504] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:30:19,506] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:30:19,508] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:30:19,510] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:30:19,512] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:30:20,939] INFO : VALIDATION | Step=30000/500000 | Train Loss=1.5656 | Valid Loss=5.5991 | Train Loss Unweighted=0.7949 | Valid Loss Unweighted=3.0657 | Stale=1 | ETA=843.4m +rank0 [2026-11-14 05:30:20,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:30:21,014] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:30:21,021] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:30:21,081] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:30:21,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:30:21,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:30:22,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:30:22,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:30:22,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:30:22,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:30:23,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:30:23,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:30:23,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:30:23,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:30:23,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:30:24,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:30:24,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:30:24,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:30:24,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:30:25,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:30:25,239] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:30:25,245] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:30:25,782] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_30000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:30:26,174] INFO : Plotted number of elements +rank0 [2026-11-14 05:30:27,377] INFO : Plotted jets +rank0 [2026-11-14 05:30:28,327] INFO : Plotted jet ratio +rank0 [2026-11-14 05:31:05,983] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 05:31:05,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:05,944] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 05:32:05,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:15,912] INFO : Step 31000/500000 rank0 | Train Loss: 3.2095 | Train Loss Unweighted: 1.3787 | Train Loss Components: Classification: 0.0080 | Classification_binary: 1.1178 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1176 | Regression_eta: 0.0056 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0988 | Classification_binary: 1.1178 | Regression_cos_phi: 0.0500 | Regression_energy: 0.7167 | Regression_eta: 0.0365 | Regression_pt: 1.1525 | Regression_sin_phi: 0.0373 | LR: 6.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5216s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0026 | max_backward: 0.0377 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1280 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2778.4 | uss_MB: 1567.3 | child_rss_MB: 22490.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:33:05,885] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 05:33:05,885] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:34:02,199] INFO : Step 32000/500000 rank0 | Train Loss: 2.2315 | Train Loss Unweighted: 1.0825 | Train Loss Components: Classification: 0.0080 | Classification_binary: 0.9182 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0759 | Regression_eta: 0.0057 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0996 | Classification_binary: 0.9182 | Regression_cos_phi: 0.0286 | Regression_energy: 0.4623 | Regression_eta: 0.0368 | Regression_pt: 0.6653 | Regression_sin_phi: 0.0207 | LR: 7.19e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5284s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0379 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1340 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.4 | uss_MB: 1567.3 | child_rss_MB: 22490.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:34:02,201] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:34:02,271] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:34:05,863] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 05:34:05,863] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:05,749] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 05:35:05,749] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:48,998] INFO : Step 33000/500000 rank0 | Train Loss: 3.5240 | Train Loss Unweighted: 1.9010 | Train Loss Components: Classification: 0.0051 | Classification_binary: 1.6693 | Regression_cos_phi: 0.0040 | Regression_energy: 0.1038 | Regression_eta: 0.0044 | Regression_pt: 0.1037 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 1.6693 | Regression_cos_phi: 0.0366 | Regression_energy: 0.6322 | Regression_eta: 0.0286 | Regression_pt: 0.9943 | Regression_sin_phi: 0.1003 | LR: 7.51e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5224s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2690.4 | uss_MB: 1479.3 | child_rss_MB: 22490.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:36:05,821] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:36:05,821] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:05,626] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:37:05,626] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:35,818] INFO : Step 34000/500000 rank0 | Train Loss: 2.6818 | Train Loss Unweighted: 2.0403 | Train Loss Components: Classification: 0.0044 | Classification_binary: 1.9510 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0344 | Regression_eta: 0.0042 | Regression_pt: 0.0342 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 1.9510 | Regression_cos_phi: 0.0270 | Regression_energy: 0.2095 | Regression_eta: 0.0275 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0836 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5420s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0891 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2690.9 | uss_MB: 1479.8 | child_rss_MB: 22491.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:37:35,820] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:37:35,889] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:38:05,820] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:38:05,820] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:05,494] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:39:05,494] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:22,868] INFO : Step 35000/500000 rank0 | Train Loss: 1.2172 | Train Loss Unweighted: 0.4525 | Train Loss Components: Classification: 0.0052 | Classification_binary: 0.3462 | Regression_cos_phi: 0.0092 | Regression_energy: 0.0374 | Regression_eta: 0.0081 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.3462 | Regression_cos_phi: 0.0836 | Regression_energy: 0.2280 | Regression_eta: 0.0528 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0569 | LR: 8.16e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5366s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1492 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2693.9 | uss_MB: 1482.8 | child_rss_MB: 22491.5 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:40:06,138] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:40:06,139] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:06,189] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:41:06,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:10,017] INFO : Step 36000/500000 rank0 | Train Loss: 1.4414 | Train Loss Unweighted: 0.9418 | Train Loss Components: Classification: 0.0014 | Classification_binary: 0.8706 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0265 | Regression_eta: 0.0064 | Regression_pt: 0.0287 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0170 | Classification_binary: 0.8706 | Regression_cos_phi: 0.0492 | Regression_energy: 0.1617 | Regression_eta: 0.0415 | Regression_pt: 0.2748 | Regression_sin_phi: 0.0265 | LR: 8.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2694.0 | uss_MB: 1482.8 | child_rss_MB: 22490.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:41:10,019] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:41:10,091] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:42:06,522] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:42:06,522] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:42:57,036] INFO : Step 37000/500000 rank0 | Train Loss: 1.4451 | Train Loss Unweighted: 0.7644 | Train Loss Components: Classification: 0.0048 | Classification_binary: 0.6716 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0309 | Regression_eta: 0.0056 | Regression_pt: 0.0387 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.6716 | Regression_cos_phi: 0.0610 | Regression_energy: 0.1884 | Regression_eta: 0.0363 | Regression_pt: 0.3712 | Regression_sin_phi: 0.0567 | LR: 8.85e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5285s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0774 | max_tensorboard_logging: 0.1256 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2694.0 | uss_MB: 1482.9 | child_rss_MB: 22492.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:43:06,838] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:43:06,838] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:07,489] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:44:07,489] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:44,767] INFO : Step 38000/500000 rank0 | Train Loss: 1.5358 | Train Loss Unweighted: 0.9550 | Train Loss Components: Classification: 0.0047 | Classification_binary: 0.8724 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0322 | Regression_eta: 0.0088 | Regression_pt: 0.0303 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.8724 | Regression_cos_phi: 0.0254 | Regression_energy: 0.1963 | Regression_eta: 0.0570 | Regression_pt: 0.2905 | Regression_sin_phi: 0.0358 | LR: 9.21e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5137s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0359 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2691.0 | uss_MB: 1479.8 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:44:44,769] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:44:44,841] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:45:07,850] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:45:07,850] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:07,978] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:46:07,978] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:32,109] INFO : Step 39000/500000 rank0 | Train Loss: 2.1355 | Train Loss Unweighted: 1.2481 | Train Loss Components: Classification: 0.0032 | Classification_binary: 1.1204 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0528 | Regression_eta: 0.0093 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 1.1204 | Regression_cos_phi: 0.0356 | Regression_energy: 0.3215 | Regression_eta: 0.0602 | Regression_pt: 0.4518 | Regression_sin_phi: 0.1068 | LR: 9.57e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5244s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0378 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2693.8 | uss_MB: 1482.6 | child_rss_MB: 22491.3 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:47:08,414] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:47:08,414] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:08,628] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:48:08,629] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:19,499] INFO : Step 40000/500000 rank0 | Train Loss: 1.8948 | Train Loss Unweighted: 0.5663 | Train Loss Components: Classification: 0.0024 | Classification_binary: 0.3738 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0851 | Regression_eta: 0.0097 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0295 | Classification_binary: 0.3738 | Regression_cos_phi: 0.0450 | Regression_energy: 0.5186 | Regression_eta: 0.0627 | Regression_pt: 0.8113 | Regression_sin_phi: 0.0539 | LR: 9.94e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5267s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0360 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1312 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2694.1 | uss_MB: 1483.0 | child_rss_MB: 22489.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 14.8 +rank0 [2026-11-14 05:48:19,501] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:48:19,569] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:48:19,570] INFO : Running validation on rank0 +rank0 [2026-11-14 05:48:19,570] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:48:19,570] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:48:19,688] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:48:19,794] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:48:19,896] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:48:19,962] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:48:20,029] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:48:20,096] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:48:20,164] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:48:20,231] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:48:20,341] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:48:20,444] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:48:20,544] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:48:20,610] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:48:20,674] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:48:20,738] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:48:20,802] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:48:20,866] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:48:20,963] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:48:20,969] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:48:20,971] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:48:20,973] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:48:20,975] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:48:20,977] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:48:20,979] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:48:20,981] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:48:20,983] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:48:20,985] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:48:20,987] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:48:20,989] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:48:20,989] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:48:20,991] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:48:20,993] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:48:20,995] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:48:22,396] INFO : VALIDATION | Step=40000/500000 | Train Loss=1.8948 | Valid Loss=7.6595 | Train Loss Unweighted=0.5663 | Valid Loss Unweighted=4.6689 | Stale=2 | ETA=826.3m +rank0 [2026-11-14 05:48:22,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:48:22,837] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 05:48:22,845] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 05:48:22,909] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:48:23,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 05:48:23,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 05:48:23,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 05:48:24,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 05:48:24,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 05:48:24,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 05:48:24,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 05:48:25,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 05:48:25,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 05:48:25,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 05:48:25,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 05:48:26,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 05:48:26,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 05:48:26,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 05:48:26,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 05:48:27,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 05:48:27,118] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:48:27,124] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:48:27,646] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_40000/cld_edm_z_tautau_pf +rank0 [2026-11-14 05:48:28,022] INFO : Plotted number of elements +rank0 [2026-11-14 05:48:29,270] INFO : Plotted jets +rank0 [2026-11-14 05:48:30,206] INFO : Plotted jet ratio +rank0 [2026-11-14 05:49:20,197] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:49:20,197] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:17,912] INFO : Step 41000/500000 rank0 | Train Loss: 1.2877 | Train Loss Unweighted: 0.4909 | Train Loss Components: Classification: 0.0039 | Classification_binary: 0.3772 | Regression_cos_phi: 0.0025 | Regression_energy: 0.0471 | Regression_eta: 0.0073 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.3772 | Regression_cos_phi: 0.0227 | Regression_energy: 0.2873 | Regression_eta: 0.0473 | Regression_pt: 0.4559 | Regression_sin_phi: 0.0490 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5279s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0028 | max_backward: 0.0386 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1362 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2715.2 | uss_MB: 1504.1 | child_rss_MB: 22490.0 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 12.8 +rank0 [2026-11-14 05:50:20,673] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:50:20,673] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:20,503] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:51:20,504] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:04,831] INFO : Step 42000/500000 rank0 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7732 | Train Loss Components: Classification: 0.0011 | Classification_binary: 0.6654 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0501 | Regression_eta: 0.0192 | Regression_pt: 0.0333 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0134 | Classification_binary: 0.6654 | Regression_cos_phi: 0.0182 | Regression_energy: 0.3050 | Regression_eta: 0.1244 | Regression_pt: 0.3190 | Regression_sin_phi: 0.0201 | LR: 1.07e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5187s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0379 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1214 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2715.7 | uss_MB: 1504.6 | child_rss_MB: 22492.1 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:52:04,832] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:52:04,903] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:52:20,859] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:52:20,859] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:53:20,892] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:53:20,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:53:52,037] INFO : Step 43000/500000 rank0 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5793 | Train Loss Components: Classification: 0.0018 | Classification_binary: 0.5048 | Regression_cos_phi: 0.0062 | Regression_energy: 0.0290 | Regression_eta: 0.0043 | Regression_pt: 0.0303 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0219 | Classification_binary: 0.5048 | Regression_cos_phi: 0.0561 | Regression_energy: 0.1766 | Regression_eta: 0.0278 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0277 | LR: 1.11e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5116s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1216 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2717.6 | uss_MB: 1506.5 | child_rss_MB: 22492.9 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 05:54:21,218] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:54:21,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:55:21,165] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:55:21,165] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:55:38,917] INFO : Step 44000/500000 rank0 | Train Loss: 1.3305 | Train Loss Unweighted: 0.7908 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.7123 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0319 | Regression_eta: 0.0068 | Regression_pt: 0.0341 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0008 | Classification_binary: 0.7123 | Regression_cos_phi: 0.0283 | Regression_energy: 0.1944 | Regression_eta: 0.0439 | Regression_pt: 0.3275 | Regression_sin_phi: 0.0233 | LR: 1.15e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5133s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0030 | max_backward: 0.0391 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1276 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2718.3 | uss_MB: 1507.2 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:55:38,919] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:55:38,987] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:56:21,085] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:56:21,085] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:20,919] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:57:20,919] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:25,647] INFO : Step 45000/500000 rank0 | Train Loss: 2.0639 | Train Loss Unweighted: 1.1701 | Train Loss Components: Classification: 0.0025 | Classification_binary: 1.0413 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0559 | Regression_eta: 0.0062 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0305 | Classification_binary: 1.0413 | Regression_cos_phi: 0.0495 | Regression_energy: 0.3409 | Regression_eta: 0.0403 | Regression_pt: 0.5156 | Regression_sin_phi: 0.0457 | LR: 1.19e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5443s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.1286 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2718.3 | uss_MB: 1507.2 | child_rss_MB: 22491.6 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:58:21,018] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:58:21,018] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:59:12,296] INFO : Step 46000/500000 rank0 | Train Loss: 1.7060 | Train Loss Unweighted: 1.1763 | Train Loss Components: Classification: 0.0026 | Classification_binary: 1.1005 | Regression_cos_phi: 0.0033 | Regression_energy: 0.0300 | Regression_eta: 0.0065 | Regression_pt: 0.0274 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 1.1005 | Regression_cos_phi: 0.0300 | Regression_energy: 0.1830 | Regression_eta: 0.0423 | Regression_pt: 0.2628 | Regression_sin_phi: 0.0548 | LR: 1.23e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5139s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0709 | max_tensorboard_logging: 0.1206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2715.7 | uss_MB: 1504.7 | child_rss_MB: 22490.7 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 05:59:12,298] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 05:59:12,369] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:59:21,265] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 05:59:21,265] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:21,050] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 06:00:21,050] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:59,191] INFO : Step 47000/500000 rank0 | Train Loss: 1.4635 | Train Loss Unweighted: 0.9193 | Train Loss Components: Classification: 0.0075 | Classification_binary: 0.8462 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0254 | Regression_eta: 0.0042 | Regression_pt: 0.0299 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0931 | Classification_binary: 0.8462 | Regression_cos_phi: 0.0399 | Regression_energy: 0.1548 | Regression_eta: 0.0274 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0158 | LR: 1.27e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5281s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0378 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1343 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2717.7 | uss_MB: 1506.6 | child_rss_MB: 22491.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 06:01:21,241] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 06:01:21,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:21,523] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 06:02:21,523] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:46,325] INFO : Step 48000/500000 rank0 | Train Loss: 1.6065 | Train Loss Unweighted: 0.7754 | Train Loss Components: Classification: 0.0053 | Classification_binary: 0.6572 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0501 | Regression_eta: 0.0066 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.6572 | Regression_cos_phi: 0.0607 | Regression_energy: 0.3050 | Regression_eta: 0.0425 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0229 | LR: 1.31e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5355s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0344 | max_loss: 0.0025 | max_backward: 0.0441 | max_optimizer_step: 0.0740 | max_tensorboard_logging: 0.1294 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2718.0 | uss_MB: 1506.9 | child_rss_MB: 22489.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 06:02:46,327] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:02:46,395] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:03:21,554] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 06:03:21,554] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:21,629] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 06:04:21,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:33,363] INFO : Step 49000/500000 rank0 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4409 | Train Loss Components: Classification: 0.0001 | Classification_binary: 0.3646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0363 | Regression_eta: 0.0029 | Regression_pt: 0.0331 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0008 | Classification_binary: 0.3646 | Regression_cos_phi: 0.0176 | Regression_energy: 0.2214 | Regression_eta: 0.0186 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0189 | LR: 1.36e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5291s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1214 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2718.0 | uss_MB: 1506.9 | child_rss_MB: 22489.2 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 18.8 +rank0 [2026-11-14 06:05:22,059] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 06:05:22,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:20,967] INFO : Step 50000/500000 rank0 | Train Loss: 1.1965 | Train Loss Unweighted: 0.5216 | Train Loss Components: Classification: 0.0022 | Classification_binary: 0.4239 | Regression_cos_phi: 0.0020 | Regression_energy: 0.0450 | Regression_eta: 0.0039 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 12.3897 | Classification_binary: 1.0000 | Regression_cos_phi: 9.1082 | Regression_energy: 6.0932 | Regression_eta: 6.4887 | Regression_pt: 9.5914 | Regression_sin_phi: 9.2716 | Task Weighted Losses: Classification: 0.0277 | Classification_binary: 0.4239 | Regression_cos_phi: 0.0181 | Regression_energy: 0.2740 | Regression_eta: 0.0251 | Regression_pt: 0.4062 | Regression_sin_phi: 0.0215 | LR: 1.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5256s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0349 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2715.4 | uss_MB: 1504.3 | child_rss_MB: 22491.8 | allocated_MB: 125.2 | reserved_MB: 248.0 | inactive_split_MB: 16.8 +rank0 [2026-11-14 06:06:20,969] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:06:21,039] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:06:21,040] INFO : Running validation on rank0 +rank0 [2026-11-14 06:06:21,040] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:06:21,040] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:06:21,170] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 06:06:21,278] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:06:21,380] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:06:21,449] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:06:21,514] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:06:21,579] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:06:21,645] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:06:21,710] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:06:21,818] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:06:21,918] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:06:22,016] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:06:22,084] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:06:22,149] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:06:22,213] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:06:22,278] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:06:22,342] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:06:22,441] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:06:22,447] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:06:22,448] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:06:22,451] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:06:22,453] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:06:22,455] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:06:22,457] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:06:22,459] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:06:22,461] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:06:22,463] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:06:22,465] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:06:22,466] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:06:22,468] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:06:22,469] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:06:22,471] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:06:22,474] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:06:23,887] INFO : VALIDATION | Step=50000/500000 | Train Loss=1.1965 | Valid Loss=9.5837 | Train Loss Unweighted=0.5216 | Valid Loss Unweighted=6.3989 | Stale=3 | ETA=808.9m +rank0 [2026-11-14 06:06:23,888] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:06:23,958] INFO : test_dataset: cld_edm_z_tautau_pf, 250 +rank0 [2026-11-14 06:06:23,967] INFO : Running predictions on cld_edm_z_tautau_pf +rank0 [2026-11-14 06:06:24,033] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:06:24,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_0.parquet +rank0 [2026-11-14 06:06:24,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_1.parquet +rank0 [2026-11-14 06:06:25,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_2.parquet +rank0 [2026-11-14 06:06:25,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_3.parquet +rank0 [2026-11-14 06:06:25,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_4.parquet +rank0 [2026-11-14 06:06:25,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_5.parquet +rank0 [2026-11-14 06:06:26,065] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_6.parquet +rank0 [2026-11-14 06:06:26,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_7.parquet +rank0 [2026-11-14 06:06:26,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_8.parquet +rank0 [2026-11-14 06:06:26,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_9.parquet +rank0 [2026-11-14 06:06:27,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_10.parquet +rank0 [2026-11-14 06:06:27,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_11.parquet +rank0 [2026-11-14 06:06:27,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_12.parquet +rank0 [2026-11-14 06:06:27,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_13.parquet +rank0 [2026-11-14 06:06:27,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_14.parquet +rank0 [2026-11-14 06:06:28,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf/pred_0_15.parquet +rank0 [2026-11-14 06:06:28,288] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:06:28,293] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 06:06:28,822] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_ecal_5p1mm_seed12345_20261114_043602_171596/preds_step_50000/cld_edm_z_tautau_pf +rank0 [2026-11-14 06:06:29,196] INFO : Plotted number of elements +rank0 [2026-11-14 06:06:30,426] INFO : Plotted jets +rank0 [2026-11-14 06:06:31,349] INFO : Plotted jet ratio +rank0 [2026-11-14 06:06:31,768] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 06:06:31,768] INFO : Training completed. Total time on device 0: 90.012min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..9e309dc5104b24fec11891c64106acf6c1d69458 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e32e4ba6991f43bf6b9deb2df2a6d6cde40fb17b844048758720adaa1fa467 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-80000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-80000.pth new file mode 100644 index 0000000000000000000000000000000000000000..73a9b2ecc8bf9910fd61e78b0cdacd533ca9c047 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-80000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45449e7e7895216a97cdece92136019fe5d3f8e959c8f15a10b310787e277766 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..fab2a6fd5ad40409fe87c90aa0f29155ea1ab54a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 3.656383752822876, "Classification": 0.03788668289780617, "Regression_pt": 0.1704087257385254, "Regression_eta": 0.014256589114665985, "Regression_sin_phi": 0.009377021342515945, "Regression_cos_phi": 0.011542309075593948, "Regression_energy": 0.17386673390865326, "Total": 8.627856254577637, "Total_unweighted": 4.073721814900637}, "train_group": {}, "valid": {"Classification_binary": 3.959590435028076, "Classification": 0.036304399371147156, "Regression_pt": 0.2332621067762375, "Regression_eta": 0.011526527814567089, "Regression_sin_phi": 0.011575575917959213, "Regression_cos_phi": 0.011714313179254532, "Regression_energy": 0.23051507771015167, "Total": 10.391823768615723, "Total_unweighted": 4.494488435797393, "metrics/particle/class_1/efficiency": 0.8949557542800903, "metrics/particle/class_1/pid_accuracy": 0.9792346358299255, "metrics/particle/class_2/efficiency": 0.19906048476696014, "metrics/particle/class_2/pid_accuracy": 0.31563422083854675, "metrics/particle/class_3/efficiency": 0.6902158260345459, "metrics/particle/class_3/pid_accuracy": 0.9754626750946045, "metrics/particle/class_4/efficiency": 0.8780487775802612, "metrics/particle/class_4/pid_accuracy": 0.007936508394777775, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.15909090638160706, "metrics/particle/count/bias_mean": -0.656000018119812, "metrics/particle/count/mae": 5.111999988555908, "metrics/particle/count/prediction_mean": 94.56400299072266, "metrics/particle/count/target_mean": 95.22000122070312, "metrics/particle/event/energy_relative_abs_error": 0.07351388782262802, "metrics/particle/event/energy_response_mean": 0.9791673421859741, "metrics/particle/event/met_abs_error": 13.202387809753418, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07453181594610214, "metrics/particle/event/scalar_pt_response_mean": 0.9817554354667664, "metrics/particle/event/vector_pt_closure": 0.08221223205327988, "metrics/particle/matched/delta_eta_abs_mean": 0.022013435140252113, "metrics/particle/matched/delta_phi_abs_mean": 0.01969347707927227, "metrics/particle/matched/delta_r_mean": 0.0328894667327404, "metrics/particle/matched/energy_relative_abs_mean": 0.09945528209209442, "metrics/particle/matched/pid_accuracy": 0.9436486959457397, "metrics/particle/matched/pt_relative_abs_mean": 0.0914686843752861, "metrics/particle/matching/duplicate_fraction": 0.044160567224025726, "metrics/particle/matching/efficiency": 0.7566477656364441, "metrics/particle/matching/f1": 0.7592631578445435, "metrics/particle/matching/purity": 0.7618967294692993, "metrics/particle/matching/target_coverage_dr0p05": 0.6205838918685913, "metrics/particle/matching/target_coverage_dr0p10": 0.7937828302383423, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6705734133720398, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8142406940460205}, "step_train_time": 0.7049527168273926, "step_valid_time": 1.8093130079796538, "step_total_time": 737.3162877559662, "elapsed_training_loop_seconds": 737.3162877559662, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9980052381266789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19345113031766092, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1491, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1315, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1235, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8790106951871658, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8819584171696848, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8804820890525611, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11804158283031523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8255347593582888, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8283031522468142, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8269166387679946, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8790106951871658, "step_test_time": 3.115822864929214, "step_plot_time": 2.3403634410351515, "step_validation_cycle_time": 7.736715724924579} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..a1015ae0b890f55a85c5e5286570aa7ecce00bfc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 3.738154888153076, "Classification": 0.03618019074201584, "Regression_pt": 0.18888677656650543, "Regression_eta": 0.009143045172095299, "Regression_sin_phi": 0.010293815284967422, "Regression_cos_phi": 0.011556721292436123, "Regression_energy": 0.1921813040971756, "Total": 9.111658096313477, "Total_unweighted": 4.186396741308272}, "train_group": {}, "valid": {"Classification_binary": 3.8092727661132812, "Classification": 0.03452149033546448, "Regression_pt": 0.21990957856178284, "Regression_eta": 0.010359999723732471, "Regression_sin_phi": 0.010734045878052711, "Regression_cos_phi": 0.01081736758351326, "Regression_energy": 0.21839606761932373, "Total": 9.879716873168945, "Total_unweighted": 4.314011315815151, "metrics/particle/class_1/efficiency": 0.8983185887336731, "metrics/particle/class_1/pid_accuracy": 0.9781302213668823, "metrics/particle/class_2/efficiency": 0.24544920027256012, "metrics/particle/class_2/pid_accuracy": 0.440191388130188, "metrics/particle/class_3/efficiency": 0.7285396456718445, "metrics/particle/class_3/pid_accuracy": 0.9790117144584656, "metrics/particle/class_4/efficiency": 0.8989546895027161, "metrics/particle/class_4/pid_accuracy": 0.06201550364494324, "metrics/particle/class_5/efficiency": 0.9780219793319702, "metrics/particle/class_5/pid_accuracy": 0.3314606845378876, "metrics/particle/count/bias_mean": 1.0520000457763672, "metrics/particle/count/mae": 4.995999813079834, "metrics/particle/count/prediction_mean": 96.27200317382812, "metrics/particle/count/target_mean": 95.22000122070312, "metrics/particle/event/energy_relative_abs_error": 0.06066230684518814, "metrics/particle/event/energy_response_mean": 0.9905630350112915, "metrics/particle/event/met_abs_error": 11.26790714263916, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06315870583057404, "metrics/particle/event/scalar_pt_response_mean": 0.9954473972320557, "metrics/particle/event/vector_pt_closure": 0.07506770640611649, "metrics/particle/matched/delta_eta_abs_mean": 0.01892063021659851, "metrics/particle/matched/delta_phi_abs_mean": 0.017917729914188385, "metrics/particle/matched/delta_r_mean": 0.02902812324464321, "metrics/particle/matched/energy_relative_abs_mean": 0.09673479944467545, "metrics/particle/matched/pid_accuracy": 0.947391152381897, "metrics/particle/matched/pt_relative_abs_mean": 0.09470311552286148, "metrics/particle/matching/duplicate_fraction": 0.04387568682432175, "metrics/particle/matching/efficiency": 0.778533935546875, "metrics/particle/matching/f1": 0.7742568850517273, "metrics/particle/matching/purity": 0.7700265645980835, "metrics/particle/matching/target_coverage_dr0p05": 0.6715396046638489, "metrics/particle/matching/target_coverage_dr0p10": 0.8131484985351562, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6870405673980713, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8300777077674866}, "step_train_time": 0.7288181781768799, "step_valid_time": 1.999717279104516, "step_total_time": 1478.442703962326, "elapsed_training_loop_seconds": 1478.442703962326, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0091367908581468, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17793659554434482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1332, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1272, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8903743315508021, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8903743315508021, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8903743315508021, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10962566844919786, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8502673796791443, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8502673796791443, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8502673796791443, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8903743315508021, "step_test_time": 3.197168626007624, "step_plot_time": 2.2347332999343053, "step_validation_cycle_time": 7.931479854974896} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..b88abb86b53663536df6eaf259ecf9d3efc46588 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 3.5233094692230225, "Classification": 0.03560318797826767, "Regression_pt": 0.15607821941375732, "Regression_eta": 0.012806667014956474, "Regression_sin_phi": 0.008343011140823364, "Regression_cos_phi": 0.009432674385607243, "Regression_energy": 0.15683439373970032, "Total": 8.043593406677246, "Total_unweighted": 3.902407622896135}, "train_group": {}, "valid": {"Classification_binary": 3.6759588718414307, "Classification": 0.033239226788282394, "Regression_pt": 0.21238690614700317, "Regression_eta": 0.010007326491177082, "Regression_sin_phi": 0.010349981486797333, "Regression_cos_phi": 0.010501870885491371, "Regression_energy": 0.2117156833410263, "Total": 9.546566009521484, "Total_unweighted": 4.164159866981208, "metrics/particle/class_1/efficiency": 0.9107964634895325, "metrics/particle/class_1/pid_accuracy": 0.9782355427742004, "metrics/particle/class_2/efficiency": 0.24486200511455536, "metrics/particle/class_2/pid_accuracy": 0.5011990666389465, "metrics/particle/class_3/efficiency": 0.735217273235321, "metrics/particle/class_3/pid_accuracy": 0.9778860211372375, "metrics/particle/class_4/efficiency": 0.891986072063446, "metrics/particle/class_4/pid_accuracy": 0.1171875, "metrics/particle/class_5/efficiency": 0.9560439586639404, "metrics/particle/class_5/pid_accuracy": 0.33908045291900635, "metrics/particle/count/bias_mean": 0.24400000274181366, "metrics/particle/count/mae": 4.796000003814697, "metrics/particle/count/prediction_mean": 95.46399688720703, "metrics/particle/count/target_mean": 95.22000122070312, "metrics/particle/event/energy_relative_abs_error": 0.05493549257516861, "metrics/particle/event/energy_response_mean": 0.9761186242103577, "metrics/particle/event/met_abs_error": 10.331265449523926, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05408588796854019, "metrics/particle/event/scalar_pt_response_mean": 0.9827778339385986, "metrics/particle/event/vector_pt_closure": 0.06621231138706207, "metrics/particle/matched/delta_eta_abs_mean": 0.018845221027731895, "metrics/particle/matched/delta_phi_abs_mean": 0.018354292958974838, "metrics/particle/matched/delta_r_mean": 0.02929048426449299, "metrics/particle/matched/energy_relative_abs_mean": 0.09571932256221771, "metrics/particle/matched/pid_accuracy": 0.9497758150100708, "metrics/particle/matched/pt_relative_abs_mean": 0.08945604413747787, "metrics/particle/matching/duplicate_fraction": 0.0404759906232357, "metrics/particle/matching/efficiency": 0.7870615124702454, "metrics/particle/matching/f1": 0.7860544323921204, "metrics/particle/matching/purity": 0.7850498557090759, "metrics/particle/matching/target_coverage_dr0p05": 0.6737239956855774, "metrics/particle/matching/target_coverage_dr0p10": 0.8179793953895569, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6982566714286804, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8391934633255005}, "step_train_time": 0.7638702392578125, "step_valid_time": 1.980862571974285, "step_total_time": 2223.2407171726227, "elapsed_training_loop_seconds": 2223.2407171726227, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9948836300840378, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1523276846307161, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1330, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1296, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8890374331550802, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8986486486486487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8938172043010753, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10135135135135136, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8663101604278075, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8756756756756757, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8709677419354839, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8890374331550802, "step_test_time": 3.2173180429963395, "step_plot_time": 2.2675488319946453, "step_validation_cycle_time": 7.982549811014906} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..99511be3654eedabbf6f299ab2ae8df15a333dbc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 3.3415255546569824, "Classification": 0.0343320295214653, "Regression_pt": 0.17733226716518402, "Regression_eta": 0.012716775760054588, "Regression_sin_phi": 0.009644454345107079, "Regression_cos_phi": 0.009247366338968277, "Regression_energy": 0.18542923033237457, "Total": 8.459205627441406, "Total_unweighted": 3.7702276781201363}, "train_group": {}, "valid": {"Classification_binary": 3.5090739727020264, "Classification": 0.029454918578267097, "Regression_pt": 0.20514503121376038, "Regression_eta": 0.009771783836185932, "Regression_sin_phi": 0.010139559395611286, "Regression_cos_phi": 0.010055281221866608, "Regression_energy": 0.20388111472129822, "Total": 9.145167350769043, "Total_unweighted": 3.977521661669016, "metrics/particle/class_1/efficiency": 0.9153982400894165, "metrics/particle/class_1/pid_accuracy": 0.9791183471679688, "metrics/particle/class_2/efficiency": 0.24955959618091583, "metrics/particle/class_2/pid_accuracy": 0.47294119000434875, "metrics/particle/class_3/efficiency": 0.7267976403236389, "metrics/particle/class_3/pid_accuracy": 0.9753661751747131, "metrics/particle/class_4/efficiency": 0.8850173950195312, "metrics/particle/class_4/pid_accuracy": 0.22834645211696625, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.5340909361839294, "metrics/particle/count/bias_mean": 1.2280000448226929, "metrics/particle/count/mae": 4.556000232696533, "metrics/particle/count/prediction_mean": 96.447998046875, "metrics/particle/count/target_mean": 95.22000122070312, "metrics/particle/event/energy_relative_abs_error": 0.05102553963661194, "metrics/particle/event/energy_response_mean": 0.9848490953445435, "metrics/particle/event/met_abs_error": 9.994537353515625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05029071122407913, "metrics/particle/event/scalar_pt_response_mean": 0.9899283051490784, "metrics/particle/event/vector_pt_closure": 0.06088346615433693, "metrics/particle/matched/delta_eta_abs_mean": 0.019910316914319992, "metrics/particle/matched/delta_phi_abs_mean": 0.017915647476911545, "metrics/particle/matched/delta_r_mean": 0.02981443889439106, "metrics/particle/matched/energy_relative_abs_mean": 0.09704870730638504, "metrics/particle/matched/pid_accuracy": 0.9517344832420349, "metrics/particle/matched/pt_relative_abs_mean": 0.09317929297685623, "metrics/particle/matching/duplicate_fraction": 0.041639018803834915, "metrics/particle/matching/efficiency": 0.7859273552894592, "metrics/particle/matching/f1": 0.7808919548988342, "metrics/particle/matching/purity": 0.7759206891059875, "metrics/particle/matching/target_coverage_dr0p05": 0.6683890223503113, "metrics/particle/matching/target_coverage_dr0p10": 0.8194916844367981, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6879227161407471, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8403276801109314}, "step_train_time": 0.7107758522033691, "step_valid_time": 2.0143255059374496, "step_total_time": 2966.231014251709, "elapsed_training_loop_seconds": 2966.231014251709, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974924487045609, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14965424147091255, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1345, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1324, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8990641711229946, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.905114401076716, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9020791415157612, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09488559892328398, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8850267379679144, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8909825033647375, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8879946344735077, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8990641711229946, "step_test_time": 3.048455724027008, "step_plot_time": 2.237534402986057, "step_validation_cycle_time": 7.8154988379683346} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..180cb15e35784592ae8ebb9747b86db75104e4fb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 3.155306339263916, "Classification": 0.0301146749407053, "Regression_pt": 0.16962884366512299, "Regression_eta": 0.016025835648179054, "Regression_sin_phi": 0.007736963219940662, "Regression_cos_phi": 0.007545409724116325, "Regression_energy": 0.1626441776752472, "Total": 7.862585067749023, "Total_unweighted": 3.5490022441372275}, "train_group": {}, "valid": {"Classification_binary": 3.3866543769836426, "Classification": 0.027965005487203598, "Regression_pt": 0.20690375566482544, "Regression_eta": 0.009699995629489422, "Regression_sin_phi": 0.009798677638173103, "Regression_cos_phi": 0.009412257932126522, "Regression_energy": 0.20601780712604523, "Total": 9.0423002243042, "Total_unweighted": 3.856451876461506, "metrics/particle/class_1/efficiency": 0.9115044474601746, "metrics/particle/class_1/pid_accuracy": 0.9754368662834167, "metrics/particle/class_2/efficiency": 0.29359951615333557, "metrics/particle/class_2/pid_accuracy": 0.5600000023841858, "metrics/particle/class_3/efficiency": 0.7356043457984924, "metrics/particle/class_3/pid_accuracy": 0.9776345491409302, "metrics/particle/class_4/efficiency": 0.8710801601409912, "metrics/particle/class_4/pid_accuracy": 0.25600001215934753, "metrics/particle/class_5/efficiency": 0.9780219793319702, "metrics/particle/class_5/pid_accuracy": 0.550561785697937, "metrics/particle/count/bias_mean": 1.559999942779541, "metrics/particle/count/mae": 4.616000175476074, "metrics/particle/count/prediction_mean": 96.77999877929688, "metrics/particle/count/target_mean": 95.22000122070312, "metrics/particle/event/energy_relative_abs_error": 0.051434025168418884, "metrics/particle/event/energy_response_mean": 0.9986711740493774, "metrics/particle/event/met_abs_error": 9.138772010803223, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04971089959144592, "metrics/particle/event/scalar_pt_response_mean": 0.9997166991233826, "metrics/particle/event/vector_pt_closure": 0.06003151834011078, "metrics/particle/matched/delta_eta_abs_mean": 0.020255832001566887, "metrics/particle/matched/delta_phi_abs_mean": 0.01784244179725647, "metrics/particle/matched/delta_r_mean": 0.030049597844481468, "metrics/particle/matched/energy_relative_abs_mean": 0.09636647999286652, "metrics/particle/matched/pid_accuracy": 0.9517233967781067, "metrics/particle/matched/pt_relative_abs_mean": 0.09186763316392899, "metrics/particle/matching/duplicate_fraction": 0.03980161249637604, "metrics/particle/matching/efficiency": 0.7909682989120483, "metrics/particle/matching/f1": 0.784541666507721, "metrics/particle/matching/purity": 0.7782186269760132, "metrics/particle/matching/target_coverage_dr0p05": 0.6694812178611755, "metrics/particle/matching/target_coverage_dr0p10": 0.8234824538230896, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6974164843559265, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8454106450080872}, "step_train_time": 0.719890832901001, "step_valid_time": 1.9710823380155489, "step_total_time": 3711.087324619293, "elapsed_training_loop_seconds": 3711.087324619293, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9993434732640497, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14154591163165398, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1475, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1331, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1311, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8897058823529411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9023728813559322, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8959946146078761, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0976271186440678, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8763368983957219, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8888135593220339, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8825311342982161, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8897058823529411, "step_test_time": 3.234652994899079, "step_plot_time": 2.244431643979624, "step_validation_cycle_time": 7.940135818091221} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_60000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..36a47f330947044ba3b2b119064b97a1642f7f17 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 106.76156583629893, "train": {"Classification_binary": 2.967411518096924, "Classification": 0.029575861990451813, "Regression_pt": 0.10179215669631958, "Regression_eta": 0.010311195626854897, "Regression_sin_phi": 0.010794652625918388, "Regression_cos_phi": 0.011392160318791866, "Regression_energy": 0.10510240495204926, "Total": 6.146047115325928, "Total_unweighted": 3.2363799503073096}, "train_group": {}, "valid": {"Classification_binary": 3.4007086753845215, "Classification": 0.0280542504042387, "Regression_pt": 0.22608496248722076, "Regression_eta": 0.009529246017336845, "Regression_sin_phi": 0.009837768040597439, "Regression_cos_phi": 0.009407778270542622, "Regression_energy": 0.22466765344142914, "Total": 9.520910263061523, "Total_unweighted": 3.908290334045887, "metrics/particle/class_1/efficiency": 0.9094690084457397, "metrics/particle/class_1/pid_accuracy": 0.9782037734985352, "metrics/particle/class_2/efficiency": 0.24897240102291107, "metrics/particle/class_2/pid_accuracy": 0.5778301954269409, "metrics/particle/class_3/efficiency": 0.7330881357192993, "metrics/particle/class_3/pid_accuracy": 0.9778217673301697, "metrics/particle/class_4/efficiency": 0.8780487775802612, "metrics/particle/class_4/pid_accuracy": 0.2817460298538208, "metrics/particle/class_5/efficiency": 0.9615384340286255, "metrics/particle/class_5/pid_accuracy": 0.5485714077949524, "metrics/particle/count/bias_mean": -2.552000045776367, "metrics/particle/count/mae": 5.28000020980835, "metrics/particle/count/prediction_mean": 92.66799926757812, "metrics/particle/count/target_mean": 95.22000122070312, "metrics/particle/event/energy_relative_abs_error": 0.07138939201831818, "metrics/particle/event/energy_response_mean": 0.9406442046165466, "metrics/particle/event/met_abs_error": 9.553435325622559, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07057912647724152, "metrics/particle/event/scalar_pt_response_mean": 0.9408341646194458, "metrics/particle/event/vector_pt_closure": 0.06231355667114258, "metrics/particle/matched/delta_eta_abs_mean": 0.020293589681386948, "metrics/particle/matched/delta_phi_abs_mean": 0.018008969724178314, "metrics/particle/matched/delta_r_mean": 0.030150387436151505, "metrics/particle/matched/energy_relative_abs_mean": 0.09450255334377289, "metrics/particle/matched/pid_accuracy": 0.9555686116218567, "metrics/particle/matched/pt_relative_abs_mean": 0.08693890273571014, "metrics/particle/matching/duplicate_fraction": 0.03254629299044609, "metrics/particle/matching/efficiency": 0.7856752872467041, "metrics/particle/matching/f1": 0.7963467836380005, "metrics/particle/matching/purity": 0.8073121309280396, "metrics/particle/matching/target_coverage_dr0p05": 0.6541062593460083, "metrics/particle/matching/target_coverage_dr0p10": 0.811636209487915, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7022054195404053, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8377231955528259}, "step_train_time": 0.7288229465484619, "step_valid_time": 2.028277160017751, "step_total_time": 4465.515907287598, "elapsed_training_loop_seconds": 4465.515907287598, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9732585141913477, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1513874931236694, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1469, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1338, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1325, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8943850267379679, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9108236895847516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9025295109612141, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08917631041524847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8856951871657754, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9019741320626277, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.893760539629005, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8943850267379679, "step_test_time": 3.150650185998529, "step_plot_time": 2.233498441055417, "step_validation_cycle_time": 7.889249885105528} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_70000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..752d088793419211665a5714d314ea3112daaa11 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 124.55516014234875, "train": {"Classification_binary": 2.8847618103027344, "Classification": 0.02030247636139393, "Regression_pt": 0.08565164357423782, "Regression_eta": 0.008092685602605343, "Regression_sin_phi": 0.00799457635730505, "Regression_cos_phi": 0.006927508860826492, "Regression_energy": 0.09073320031166077, "Total": 5.488339424133301, "Total_unweighted": 3.104463901370764}, "train_group": {}, "valid": {"Classification_binary": 3.487535238265991, "Classification": 0.028894947841763496, "Regression_pt": 0.22823399305343628, "Regression_eta": 0.009565852582454681, "Regression_sin_phi": 0.009685867466032505, "Regression_cos_phi": 0.0092926612123847, "Regression_energy": 0.2270759493112564, "Total": 9.66984748840332, "Total_unweighted": 4.000284509733319, "metrics/particle/class_1/efficiency": 0.9141592979431152, "metrics/particle/class_1/pid_accuracy": 0.9737657308578491, "metrics/particle/class_2/efficiency": 0.30710509419441223, "metrics/particle/class_2/pid_accuracy": 0.5869981050491333, "metrics/particle/class_3/efficiency": 0.735507607460022, "metrics/particle/class_3/pid_accuracy": 0.973421037197113, "metrics/particle/class_4/efficiency": 0.8815330862998962, "metrics/particle/class_4/pid_accuracy": 0.2964426875114441, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.5681818127632141, "metrics/particle/count/bias_mean": 2.0280001163482666, "metrics/particle/count/mae": 4.771999835968018, "metrics/particle/count/prediction_mean": 97.24800109863281, "metrics/particle/count/target_mean": 95.22000122070312, "metrics/particle/event/energy_relative_abs_error": 0.05403196066617966, "metrics/particle/event/energy_response_mean": 1.014094591140747, "metrics/particle/event/met_abs_error": 9.155014991760254, "metrics/particle/event/scalar_pt_relative_abs_error": 0.052261389791965485, "metrics/particle/event/scalar_pt_response_mean": 1.0185632705688477, "metrics/particle/event/vector_pt_closure": 0.05983187258243561, "metrics/particle/matched/delta_eta_abs_mean": 0.020002637058496475, "metrics/particle/matched/delta_phi_abs_mean": 0.01759730465710163, "metrics/particle/matched/delta_r_mean": 0.02963152341544628, "metrics/particle/matched/energy_relative_abs_mean": 0.09622799605131149, "metrics/particle/matched/pid_accuracy": 0.9500582814216614, "metrics/particle/matched/pt_relative_abs_mean": 0.08962789177894592, "metrics/particle/matching/duplicate_fraction": 0.04092629253864288, "metrics/particle/matching/efficiency": 0.7931947112083435, "metrics/particle/matching/f1": 0.7848369479179382, "metrics/particle/matching/purity": 0.776653528213501, "metrics/particle/matching/target_coverage_dr0p05": 0.6652383804321289, "metrics/particle/matching/target_coverage_dr0p10": 0.8253728151321411, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.703927755355835, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8500314950942993}, "step_train_time": 0.7107551097869873, "step_valid_time": 1.9751392480684444, "step_total_time": 5216.817139863968, "elapsed_training_loop_seconds": 5216.817139863968, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.011797180209774, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14509154859882645, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1500, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1352, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1331, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9037433155080213, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9013333333333333, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9025367156208278, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09866666666666667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8897058823529411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8873333333333333, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8885180240320427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9037433155080213, "step_test_time": 3.100644001038745, "step_plot_time": 2.258938900078647, "step_validation_cycle_time": 7.792087974026799} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_80000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..04f594aa9ff8561fa572d6231e6912b56cda7084 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 142.34875444839858, "train": {"Classification_binary": 2.423652172088623, "Classification": 0.02147861011326313, "Regression_pt": 0.07675126194953918, "Regression_eta": 0.006886899936944246, "Regression_sin_phi": 0.005866581108421087, "Regression_cos_phi": 0.005724621936678886, "Regression_energy": 0.08049444109201431, "Total": 4.761922836303711, "Total_unweighted": 2.620854588225484}, "train_group": {}, "valid": {"Classification_binary": 3.814948320388794, "Classification": 0.030756989493966103, "Regression_pt": 0.2572501599788666, "Regression_eta": 0.009789850562810898, "Regression_sin_phi": 0.009710975922644138, "Regression_cos_phi": 0.009406906552612782, "Regression_energy": 0.2571410536766052, "Total": 10.743698120117188, "Total_unweighted": 4.3890042565763, "metrics/particle/class_1/efficiency": 0.9112389087677002, "metrics/particle/class_1/pid_accuracy": 0.9751383662223816, "metrics/particle/class_2/efficiency": 0.26952436566352844, "metrics/particle/class_2/pid_accuracy": 0.5729847550392151, "metrics/particle/class_3/efficiency": 0.7160553336143494, "metrics/particle/class_3/pid_accuracy": 0.9754020571708679, "metrics/particle/class_4/efficiency": 0.8780487775802612, "metrics/particle/class_4/pid_accuracy": 0.2936508059501648, "metrics/particle/class_5/efficiency": 0.9615384340286255, "metrics/particle/class_5/pid_accuracy": 0.6057142615318298, "metrics/particle/count/bias_mean": -0.6759999990463257, "metrics/particle/count/mae": 4.892000198364258, "metrics/particle/count/prediction_mean": 94.54399871826172, "metrics/particle/count/target_mean": 95.22000122070312, "metrics/particle/event/energy_relative_abs_error": 0.059461768716573715, "metrics/particle/event/energy_response_mean": 0.9716150164604187, "metrics/particle/event/met_abs_error": 9.043349266052246, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05637037754058838, "metrics/particle/event/scalar_pt_response_mean": 0.9730385541915894, "metrics/particle/event/vector_pt_closure": 0.06030677258968353, "metrics/particle/matched/delta_eta_abs_mean": 0.01904325559735298, "metrics/particle/matched/delta_phi_abs_mean": 0.017702825367450714, "metrics/particle/matched/delta_r_mean": 0.0289943628013134, "metrics/particle/matched/energy_relative_abs_mean": 0.09051413834095001, "metrics/particle/matched/pid_accuracy": 0.9525884985923767, "metrics/particle/matched/pt_relative_abs_mean": 0.08444546908140182, "metrics/particle/matching/duplicate_fraction": 0.03710441663861275, "metrics/particle/matching/efficiency": 0.7805923223495483, "metrics/particle/matching/f1": 0.7833730578422546, "metrics/particle/matching/purity": 0.7861736416816711, "metrics/particle/matching/target_coverage_dr0p05": 0.6533081531524658, "metrics/particle/matching/target_coverage_dr0p10": 0.8070993423461914, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.701827347278595, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8378912210464478}, "step_train_time": 0.7258956432342529, "step_valid_time": 1.9691625200212002, "step_total_time": 5965.381831645966, "elapsed_training_loop_seconds": 5965.381831645966, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9894647395915223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14189280715414776, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1345, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1329, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8990641711229946, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9069453809844908, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9029875797247399, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0930546190155091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8883689839572193, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8961564396493594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.892245720040282, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8990641711229946, "step_test_time": 3.0850646040635183, "step_plot_time": 2.259558166959323, "step_validation_cycle_time": 7.790775749948807} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..77697bbb833954ab1e0fa1d6e233388cfd82ee5b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_10mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7f992676210632ebb533a0d4a83bb9277e01bfb9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:060af5ef21c6740a660b4010e3811647655a7d0c769fac8d1affc846f1956978 +size 4394 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..356b712e8a270e6302f500331b8ea5ffa2204d61 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..f0dadef6be005d963b1ed729c8ceb986247482aa Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0985562768719efc3b2d4ebc38556129c098d6b5 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..ac8e83799e7fce112dd40448c58001118fd3f23b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dd67dc05ebe44d9069911649fb4b3e1ea3bf1440 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..d9606a448c34549c5b9a5bc979459ad7243f835e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c857a0594f4a80dd7a3b27080ec83900f056c754 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..11cc3bcda022e364def90132375eba4767b0823f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..96a1e8e8647ee72e48afdf68cdf6ec2ac78ea206 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..5259b73621c78dc77d91846c03412dbe431630b0 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..38662f011812d868a65334175bb33ad313fd3cfd Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..1c46b405890732977ab3b59943591df021ab414c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/plots_step_80000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/runs/train/events.out.tfevents.1794650590.workergpu215.990265.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/runs/train/events.out.tfevents.1794650590.workergpu215.990265.0 new file mode 100644 index 0000000000000000000000000000000000000000..ff56eff6304ac0e3f9810cb6345d9c611ec4a673 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/runs/train/events.out.tfevents.1794650590.workergpu215.990265.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adfa1ee80827902806e940f4b7233796650820fb527f7b29125497608afda124 +size 2953648 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/runs/valid/events.out.tfevents.1794650590.workergpu215.990265.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/runs/valid/events.out.tfevents.1794650590.workergpu215.990265.1 new file mode 100644 index 0000000000000000000000000000000000000000..e598f814ef6864d37fff5ea6e2d3f86d294891d2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/runs/valid/events.out.tfevents.1794650590.workergpu215.990265.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f93e8248b9074612b581e3320ecdfcf1408fc4d0406cdc0a34e2df37e01a11e +size 4016969 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..9c2da42555735f2e2dc98cbf68a154e35cb1427d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_10mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_10mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_10mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_10mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b4c6aebaebbe3bf798da8ac5a1ef76f7d4dc7c5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_10mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8361d93f44f76d6ebc6837e8213b7395b3ab1202 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/train.log @@ -0,0 +1,1353 @@ +rank0 [2026-11-14 05:03:08,861] INFO : Will use single-gpu: NVIDIA H100 +rank0 [2026-11-14 05:03:08,861] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 05:03:09,085] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:03:09,085] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:03:09,085] INFO : Instantiating model +rank0 [2026-11-14 05:03:09,103] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:03:09,103] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:03:09,104] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:03:09,105] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:03:09,105] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:03:09,105] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:03:09,105] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,107] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:03:09,107] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,109] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:03:09,109] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,111] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:03:09,112] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,114] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:03:09,114] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,116] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:03:09,116] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,119] INFO : Backbone initialization took 0.01s +rank0 [2026-11-14 05:03:09,119] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:03:09,119] INFO : Initializing output DNNs +rank0 [2026-11-14 05:03:09,120] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,122] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,129] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:03:09,129] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:03:09,130] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:03:09,130] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:03:09,130] INFO : task_queries enabled=True +rank0 [2026-11-14 05:03:09,130] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:03:09,130] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:03:09,130] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:03:09,130] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:03:09,130] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:03:09,130] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:03:09,130] INFO : output_mode=elementwise +rank0 [2026-11-14 05:03:09,130] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:03:09,130] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:03:09,130] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:03:09,130] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:03:09,130] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:03:09,130] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:03:09,130] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:03:09,130] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:03:09,131] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:03:09,131] INFO : MLPF __init__ done +rank0 [2026-11-14 05:03:09,131] INFO : Instantiated model +rank0 [2026-11-14 05:03:09,131] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:03:09,137] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:03:09,138] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:03:09,198] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:03:09,198] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:03:09,199] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:03:09,199] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:03:09,199] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:03:09,201] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:03:09,202] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935 +rank0 [2026-11-14 05:03:09,202] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935 +rank0 [2026-11-14 05:03:09,202] INFO : Getting dataloaders +rank0 [2026-11-14 05:03:09,202] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:03:09,361] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,399] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,455] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,492] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,525] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,571] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,613] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,659] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,695] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,746] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,746] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:03:09,746] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:03:09,746] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:03:09,746] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:03:09,756] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,766] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,775] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,784] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,794] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,803] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,813] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,822] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,832] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,841] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,841] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:03:09,842] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:03:09,871] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:03:10,021] INFO : Got dataloaders +rank0 [2026-11-14 05:03:10,022] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:03:10,022] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:03:10,022] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:03:10,025] INFO : Creating train_iterator +rank0 [2026-11-14 05:03:10,025] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:10,052] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:03:10,199] INFO : Created train_iterator +rank0 [2026-11-14 05:03:18,680] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.150242385864257, 'Classification': 0.6509133404493332, 'Regression_pt': 0.5197731229662895, 'Regression_eta': 0.9022421351075173, 'Regression_sin_phi': 0.6643005856871604, 'Regression_cos_phi': 0.6540828502178192, 'Regression_energy': 0.6644598841667175}, weights={'Classification_binary': 1.0, 'Classification': 10.98493766784668, 'Regression_pt': 13.756467819213867, 'Regression_eta': 7.924970626831055, 'Regression_sin_phi': 10.763565063476562, 'Regression_cos_phi': 10.931707382202148, 'Regression_energy': 10.760984420776367} +rank0 [2026-11-14 05:03:52,570] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:03:52,570] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:24,547] INFO : Step 1000/500000 rank0 | Train Loss: 12.6559 | Train Loss Unweighted: 5.1529 | Train Loss Components: Classification: 0.0637 | Classification_binary: 4.4608 | Regression_cos_phi: 0.0216 | Regression_energy: 0.2681 | Regression_eta: 0.0365 | Regression_pt: 0.2779 | Regression_sin_phi: 0.0244 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.6997 | Classification_binary: 4.4608 | Regression_cos_phi: 0.2358 | Regression_energy: 2.8856 | Regression_eta: 0.2896 | Regression_pt: 3.8224 | Regression_sin_phi: 0.2621 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7175s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0021 | max_backward: 0.0254 | max_optimizer_step: 0.0773 | max_tensorboard_logging: 0.2240 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2601.1 | uss_MB: 1351.0 | child_rss_MB: 22759.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:04:33,597] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:04:33,598] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:14,686] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:05:14,687] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:38,256] INFO : Step 2000/500000 rank0 | Train Loss: 11.3900 | Train Loss Unweighted: 4.8544 | Train Loss Components: Classification: 0.0438 | Classification_binary: 4.2578 | Regression_cos_phi: 0.0157 | Regression_energy: 0.2469 | Regression_eta: 0.0202 | Regression_pt: 0.2528 | Regression_sin_phi: 0.0173 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4811 | Classification_binary: 4.2578 | Regression_cos_phi: 0.1714 | Regression_energy: 2.6565 | Regression_eta: 0.1599 | Regression_pt: 3.4772 | Regression_sin_phi: 0.1861 | LR: 2.02e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7101s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0233 | max_loss: 0.0022 | max_backward: 0.0272 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2603.3 | uss_MB: 1352.4 | child_rss_MB: 22760.9 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:05:38,257] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:05:56,449] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:05:56,449] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:37,847] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:06:37,847] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:52,384] INFO : Step 3000/500000 rank0 | Train Loss: 10.9199 | Train Loss Unweighted: 4.8681 | Train Loss Components: Classification: 0.0405 | Classification_binary: 4.3168 | Regression_cos_phi: 0.0165 | Regression_energy: 0.2271 | Regression_eta: 0.0173 | Regression_pt: 0.2361 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4447 | Classification_binary: 4.3168 | Regression_cos_phi: 0.1803 | Regression_energy: 2.4433 | Regression_eta: 0.1374 | Regression_pt: 3.2477 | Regression_sin_phi: 0.1498 | LR: 2.05e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7104s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0022 | max_backward: 0.0263 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2253 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2606.8 | uss_MB: 1356.1 | child_rss_MB: 22760.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:07:19,567] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:07:19,567] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:00,469] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:08:00,469] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:05,820] INFO : Step 4000/500000 rank0 | Train Loss: 9.5930 | Train Loss Unweighted: 4.6482 | Train Loss Components: Classification: 0.0418 | Classification_binary: 4.1931 | Regression_cos_phi: 0.0154 | Regression_energy: 0.1798 | Regression_eta: 0.0207 | Regression_pt: 0.1832 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4590 | Classification_binary: 4.1931 | Regression_cos_phi: 0.1680 | Regression_energy: 1.9349 | Regression_eta: 0.1643 | Regression_pt: 2.5203 | Regression_sin_phi: 0.1535 | LR: 2.08e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.6956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0233 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.2126 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2606.8 | uss_MB: 1356.2 | child_rss_MB: 22764.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:08:05,822] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:08:41,286] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:08:41,286] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:18,490] INFO : Step 5000/500000 rank0 | Train Loss: 9.1201 | Train Loss Unweighted: 4.4488 | Train Loss Components: Classification: 0.0403 | Classification_binary: 4.0176 | Regression_cos_phi: 0.0154 | Regression_energy: 0.1712 | Regression_eta: 0.0192 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0164 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4429 | Classification_binary: 4.0176 | Regression_cos_phi: 0.1679 | Regression_energy: 1.8417 | Regression_eta: 0.1519 | Regression_pt: 2.3218 | Regression_sin_phi: 0.1763 | LR: 2.13e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7067s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2104 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2635.0 | uss_MB: 1384.3 | child_rss_MB: 22763.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:09:22,692] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:09:22,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:03,423] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:10:03,423] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:31,202] INFO : Step 6000/500000 rank0 | Train Loss: 9.5896 | Train Loss Unweighted: 4.3296 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.8537 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2027 | Regression_eta: 0.0121 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3468 | Classification_binary: 3.8537 | Regression_cos_phi: 0.0885 | Regression_energy: 2.1808 | Regression_eta: 0.0957 | Regression_pt: 2.9437 | Regression_sin_phi: 0.0803 | LR: 2.19e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7075s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0770 | max_tensorboard_logging: 0.2187 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2635.0 | uss_MB: 1384.4 | child_rss_MB: 22765.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:10:31,204] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:10:44,376] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:10:44,376] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:25,345] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:11:25,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:44,726] INFO : Step 7000/500000 rank0 | Train Loss: 9.2235 | Train Loss Unweighted: 4.1558 | Train Loss Components: Classification: 0.0354 | Classification_binary: 3.6979 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1947 | Regression_eta: 0.0078 | Regression_pt: 0.2041 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3890 | Classification_binary: 3.6979 | Regression_cos_phi: 0.0832 | Regression_energy: 2.0953 | Regression_eta: 0.0615 | Regression_pt: 2.8077 | Regression_sin_phi: 0.0888 | LR: 2.26e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6980s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0769 | max_tensorboard_logging: 0.2091 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2635.5 | uss_MB: 1384.9 | child_rss_MB: 22764.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:12:06,967] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:12:06,967] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:48,285] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:12:48,285] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:58,444] INFO : Step 8000/500000 rank0 | Train Loss: 9.2284 | Train Loss Unweighted: 4.3518 | Train Loss Components: Classification: 0.0378 | Classification_binary: 3.9086 | Regression_cos_phi: 0.0158 | Regression_energy: 0.1838 | Regression_eta: 0.0073 | Regression_pt: 0.1868 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4155 | Classification_binary: 3.9086 | Regression_cos_phi: 0.1725 | Regression_energy: 1.9777 | Regression_eta: 0.0582 | Regression_pt: 2.5694 | Regression_sin_phi: 0.1265 | LR: 2.34e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7025s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0025 | max_backward: 0.0260 | max_optimizer_step: 0.0778 | max_tensorboard_logging: 0.2096 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2635.5 | uss_MB: 1384.9 | child_rss_MB: 22764.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:12:58,446] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:12:58,499] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:13:29,977] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:13:29,977] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:14:10,886] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:14:10,886] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:14:12,121] INFO : Step 9000/500000 rank0 | Train Loss: 10.6784 | Train Loss Unweighted: 4.7343 | Train Loss Components: Classification: 0.0450 | Classification_binary: 4.1951 | Regression_cos_phi: 0.0104 | Regression_energy: 0.2275 | Regression_eta: 0.0117 | Regression_pt: 0.2342 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4947 | Classification_binary: 4.1951 | Regression_cos_phi: 0.1141 | Regression_energy: 2.4477 | Regression_eta: 0.0926 | Regression_pt: 3.2218 | Regression_sin_phi: 0.1125 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2208 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2604.8 | uss_MB: 1354.1 | child_rss_MB: 22766.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 22.3 +rank0 [2026-11-14 05:14:52,270] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:14:52,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:25,476] INFO : Step 10000/500000 rank0 | Train Loss: 8.6279 | Train Loss Unweighted: 4.0737 | Train Loss Components: Classification: 0.0379 | Classification_binary: 3.6564 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1739 | Regression_eta: 0.0143 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4162 | Classification_binary: 3.6564 | Regression_cos_phi: 0.1262 | Regression_energy: 1.8710 | Regression_eta: 0.1130 | Regression_pt: 2.3442 | Regression_sin_phi: 0.1009 | LR: 2.52e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7042s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0020 | max_backward: 0.0254 | max_optimizer_step: 0.0771 | max_tensorboard_logging: 0.2119 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2605.3 | uss_MB: 1354.7 | child_rss_MB: 22766.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:15:25,477] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:15:25,523] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:15:25,524] INFO : Running validation on rank0 +rank0 [2026-11-14 05:15:25,524] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:15:25,524] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:15:25,647] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:15:25,765] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:15:25,821] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:15:25,861] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:15:25,903] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:15:25,936] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:15:25,975] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:15:26,015] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:15:26,051] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:15:26,105] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:15:26,159] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:15:26,197] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:15:26,236] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:15:26,269] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:15:26,308] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:15:26,347] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:15:26,383] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:15:26,386] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:15:26,387] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:15:26,389] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:15:26,390] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:15:26,392] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:15:26,394] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:15:26,395] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:15:26,397] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:15:26,398] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:15:26,400] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:15:26,401] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:15:26,403] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:15:26,404] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:15:26,406] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:15:26,408] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:15:27,391] INFO : VALIDATION | Step=10000/500000 | Train Loss=8.6279 | Valid Loss=10.3918 | Train Loss Unweighted=4.0737 | Valid Loss Unweighted=4.4945 | Stale=0 | ETA=602.2m +rank0 [2026-11-14 05:15:27,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:15:27,488] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:15:27,496] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:15:27,539] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:15:28,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:15:28,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:15:28,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:15:28,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:15:28,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:15:28,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:15:28,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:15:29,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:15:29,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:15:29,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:15:29,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:15:29,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:15:29,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:15:30,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:15:30,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:15:30,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:15:30,504] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:15:30,512] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:15:30,894] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:15:31,199] INFO : Plotted number of elements +rank0 [2026-11-14 05:15:32,156] INFO : Plotted jets +rank0 [2026-11-14 05:15:32,840] INFO : Plotted jet ratio +rank0 [2026-11-14 05:15:41,484] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:15:41,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:21,314] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:16:21,315] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:45,127] INFO : Step 11000/500000 rank0 | Train Loss: 9.5722 | Train Loss Unweighted: 4.1320 | Train Loss Components: Classification: 0.0387 | Classification_binary: 3.6402 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2130 | Regression_eta: 0.0062 | Regression_pt: 0.2159 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4246 | Classification_binary: 3.6402 | Regression_cos_phi: 0.1183 | Regression_energy: 2.2921 | Regression_eta: 0.0489 | Regression_pt: 2.9705 | Regression_sin_phi: 0.0775 | LR: 2.63e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7629s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.2202 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.4 | uss_MB: 1522.1 | child_rss_MB: 22767.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:17:02,665] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:17:02,665] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:43,724] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:17:43,724] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:58,923] INFO : Step 12000/500000 rank0 | Train Loss: 9.1185 | Train Loss Unweighted: 4.2970 | Train Loss Components: Classification: 0.0389 | Classification_binary: 3.8553 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1769 | Regression_eta: 0.0156 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0185 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4274 | Classification_binary: 3.8553 | Regression_cos_phi: 0.1142 | Regression_energy: 1.9035 | Regression_eta: 0.1237 | Regression_pt: 2.4949 | Regression_sin_phi: 0.1995 | LR: 2.75e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7159s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2161 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.4 | uss_MB: 1522.2 | child_rss_MB: 22769.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:17:58,925] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:17:58,975] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:18:25,322] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:18:25,322] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:06,367] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:19:06,367] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:12,314] INFO : Step 13000/500000 rank0 | Train Loss: 12.5156 | Train Loss Unweighted: 4.4627 | Train Loss Components: Classification: 0.0375 | Classification_binary: 3.7303 | Regression_cos_phi: 0.0081 | Regression_energy: 0.3307 | Regression_eta: 0.0250 | Regression_pt: 0.3218 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4123 | Classification_binary: 3.7303 | Regression_cos_phi: 0.0890 | Regression_energy: 3.5585 | Regression_eta: 0.1981 | Regression_pt: 4.4275 | Regression_sin_phi: 0.0999 | LR: 2.88e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0260 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0781 | max_tensorboard_logging: 0.2205 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2706.8 | uss_MB: 1464.6 | child_rss_MB: 22768.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:19:47,338] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:19:47,338] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:25,476] INFO : Step 14000/500000 rank0 | Train Loss: 9.9797 | Train Loss Unweighted: 4.3846 | Train Loss Components: Classification: 0.0338 | Classification_binary: 3.8791 | Regression_cos_phi: 0.0136 | Regression_energy: 0.2169 | Regression_eta: 0.0081 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3715 | Classification_binary: 3.8791 | Regression_cos_phi: 0.1486 | Regression_energy: 2.3344 | Regression_eta: 0.0641 | Regression_pt: 3.0967 | Regression_sin_phi: 0.0852 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7151s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2172 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2707.1 | uss_MB: 1464.8 | child_rss_MB: 22769.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:20:25,478] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:20:25,525] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:20:29,206] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 05:20:29,206] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:10,267] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 05:21:10,267] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:39,564] INFO : Step 15000/500000 rank0 | Train Loss: 10.0383 | Train Loss Unweighted: 4.5144 | Train Loss Components: Classification: 0.0351 | Classification_binary: 4.0122 | Regression_cos_phi: 0.0103 | Regression_energy: 0.2171 | Regression_eta: 0.0118 | Regression_pt: 0.2156 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3861 | Classification_binary: 4.0122 | Regression_cos_phi: 0.1125 | Regression_energy: 2.3365 | Regression_eta: 0.0936 | Regression_pt: 2.9662 | Regression_sin_phi: 0.1312 | LR: 3.17e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7239s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.2194 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.3 | uss_MB: 1468.1 | child_rss_MB: 22766.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:21:52,171] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 05:21:52,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:32,967] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 05:22:32,967] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:52,786] INFO : Step 16000/500000 rank0 | Train Loss: 9.1830 | Train Loss Unweighted: 4.3111 | Train Loss Components: Classification: 0.0416 | Classification_binary: 3.8618 | Regression_cos_phi: 0.0205 | Regression_energy: 0.1841 | Regression_eta: 0.0182 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4566 | Classification_binary: 3.8618 | Regression_cos_phi: 0.2246 | Regression_energy: 1.9815 | Regression_eta: 0.1444 | Regression_pt: 2.4102 | Regression_sin_phi: 0.1039 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0255 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.2198 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.6 | uss_MB: 1468.3 | child_rss_MB: 22768.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 26.3 +rank0 [2026-11-14 05:22:52,787] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:22:52,835] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:23:13,726] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 05:23:13,726] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:23:54,082] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 05:23:54,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:04,954] INFO : Step 17000/500000 rank0 | Train Loss: 9.4302 | Train Loss Unweighted: 4.1930 | Train Loss Components: Classification: 0.0431 | Classification_binary: 3.7172 | Regression_cos_phi: 0.0131 | Regression_energy: 0.1906 | Regression_eta: 0.0101 | Regression_pt: 0.2037 | Regression_sin_phi: 0.0151 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4731 | Classification_binary: 3.7172 | Regression_cos_phi: 0.1435 | Regression_energy: 2.0511 | Regression_eta: 0.0798 | Regression_pt: 2.8029 | Regression_sin_phi: 0.1627 | LR: 3.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7224s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0785 | max_tensorboard_logging: 0.2151 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2706.1 | uss_MB: 1463.9 | child_rss_MB: 22768.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:24:36,096] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 05:24:36,096] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:17,189] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 05:25:17,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:19,027] INFO : Step 18000/500000 rank0 | Train Loss: 9.0094 | Train Loss Unweighted: 4.3172 | Train Loss Components: Classification: 0.0337 | Classification_binary: 3.8911 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1811 | Regression_eta: 0.0102 | Regression_pt: 0.1846 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3697 | Classification_binary: 3.8911 | Regression_cos_phi: 0.0875 | Regression_energy: 1.9493 | Regression_eta: 0.0808 | Regression_pt: 2.5393 | Regression_sin_phi: 0.0918 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7192s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0815 | max_tensorboard_logging: 0.2149 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2707.1 | uss_MB: 1464.8 | child_rss_MB: 22769.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:25:19,029] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:25:19,077] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:25:58,889] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 05:25:58,889] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:32,660] INFO : Step 19000/500000 rank0 | Train Loss: 9.8895 | Train Loss Unweighted: 4.3911 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.8916 | Regression_cos_phi: 0.0101 | Regression_energy: 0.2145 | Regression_eta: 0.0137 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3767 | Classification_binary: 3.8916 | Regression_cos_phi: 0.1101 | Regression_energy: 2.3081 | Regression_eta: 0.1084 | Regression_pt: 2.9964 | Regression_sin_phi: 0.0983 | LR: 3.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7240s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0261 | max_loss: 0.0021 | max_backward: 0.0268 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.2182 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.3 | uss_MB: 1468.1 | child_rss_MB: 22770.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 05:26:40,543] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 05:26:40,543] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:21,395] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 05:27:21,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:46,416] INFO : Step 20000/500000 rank0 | Train Loss: 9.1117 | Train Loss Unweighted: 4.1864 | Train Loss Components: Classification: 0.0362 | Classification_binary: 3.7382 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1922 | Regression_eta: 0.0091 | Regression_pt: 0.1889 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3974 | Classification_binary: 3.7382 | Regression_cos_phi: 0.1263 | Regression_energy: 2.0681 | Regression_eta: 0.0725 | Regression_pt: 2.5984 | Regression_sin_phi: 0.1108 | LR: 4.07e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7276s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2204 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.6 | uss_MB: 1468.4 | child_rss_MB: 22767.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:27:46,418] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:27:46,465] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:27:46,466] INFO : Running validation on rank0 +rank0 [2026-11-14 05:27:46,466] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:27:46,466] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:27:46,571] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 05:27:46,654] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:27:46,733] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:27:46,781] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:27:46,826] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:27:46,870] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:27:46,916] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:27:46,962] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:27:47,037] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:27:47,115] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:27:47,191] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:27:47,238] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:27:47,284] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:27:47,328] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:27:47,373] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:27:47,419] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:27:47,486] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:27:47,491] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:27:47,492] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:27:47,495] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:27:47,495] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:27:47,497] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:27:47,499] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:27:47,501] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:27:47,502] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:27:47,503] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:27:47,505] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:27:47,506] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:27:47,508] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:27:47,509] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:27:47,511] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:27:47,512] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:27:48,516] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.1117 | Valid Loss=9.8797 | Train Loss Unweighted=4.1864 | Valid Loss Unweighted=4.3140 | Stale=0 | ETA=591.4m +rank0 [2026-11-14 05:27:48,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:27:48,614] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:27:48,623] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:27:48,686] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:27:49,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:27:49,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:27:49,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:27:49,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:27:49,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:27:50,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:27:50,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:27:50,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:27:50,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:27:50,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:27:50,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:27:50,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:27:51,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:27:51,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:27:51,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:27:51,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:27:51,711] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:27:51,718] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:27:52,092] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:27:52,333] INFO : Plotted number of elements +rank0 [2026-11-14 05:27:53,246] INFO : Plotted jets +rank0 [2026-11-14 05:27:53,942] INFO : Plotted jet ratio +rank0 [2026-11-14 05:28:11,230] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 05:28:11,230] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:52,118] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 05:28:52,118] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:29:07,774] INFO : Step 21000/500000 rank0 | Train Loss: 8.5265 | Train Loss Unweighted: 4.1585 | Train Loss Components: Classification: 0.0376 | Classification_binary: 3.7615 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1655 | Regression_eta: 0.0072 | Regression_pt: 0.1681 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4128 | Classification_binary: 3.7615 | Regression_cos_phi: 0.0911 | Regression_energy: 1.7812 | Regression_eta: 0.0573 | Regression_pt: 2.3121 | Regression_sin_phi: 0.1105 | LR: 4.28e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7357s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0005 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.2220 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2738.5 | uss_MB: 1496.4 | child_rss_MB: 22773.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:29:33,686] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 05:29:33,687] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:14,688] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 05:30:14,689] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:21,344] INFO : Step 22000/500000 rank0 | Train Loss: 7.8826 | Train Loss Unweighted: 3.9374 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.5808 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1513 | Regression_eta: 0.0053 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3243 | Classification_binary: 3.5808 | Regression_cos_phi: 0.0755 | Regression_energy: 1.6284 | Regression_eta: 0.0417 | Regression_pt: 2.1684 | Regression_sin_phi: 0.0635 | LR: 4.50e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7405s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0032 | max_backward: 0.0267 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2223 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2739.6 | uss_MB: 1497.4 | child_rss_MB: 22773.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:30:21,345] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:30:21,394] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:30:56,371] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 05:30:56,371] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:31:35,451] INFO : Step 23000/500000 rank0 | Train Loss: 9.1060 | Train Loss Unweighted: 4.0337 | Train Loss Components: Classification: 0.0303 | Classification_binary: 3.5758 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1932 | Regression_eta: 0.0078 | Regression_pt: 0.2057 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3333 | Classification_binary: 3.5758 | Regression_cos_phi: 0.1155 | Regression_energy: 2.0794 | Regression_eta: 0.0621 | Regression_pt: 2.8302 | Regression_sin_phi: 0.1096 | LR: 4.73e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7296s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0320 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2196 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2745.1 | uss_MB: 1502.9 | child_rss_MB: 22770.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:31:38,514] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 05:31:38,514] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:19,680] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 05:32:19,680] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:49,064] INFO : Step 24000/500000 rank0 | Train Loss: 8.4620 | Train Loss Unweighted: 4.0145 | Train Loss Components: Classification: 0.0306 | Classification_binary: 3.6104 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1708 | Regression_eta: 0.0094 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3357 | Classification_binary: 3.6104 | Regression_cos_phi: 0.1202 | Regression_energy: 1.8378 | Regression_eta: 0.0747 | Regression_pt: 2.3953 | Regression_sin_phi: 0.0878 | LR: 4.97e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7191s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0791 | max_tensorboard_logging: 0.2146 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2745.3 | uss_MB: 1503.1 | child_rss_MB: 22773.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:32:49,066] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:32:49,112] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:33:01,218] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 05:33:01,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:33:42,350] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 05:33:42,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:34:02,862] INFO : Step 25000/500000 rank0 | Train Loss: 9.0820 | Train Loss Unweighted: 4.1736 | Train Loss Components: Classification: 0.0388 | Classification_binary: 3.7257 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1845 | Regression_eta: 0.0146 | Regression_pt: 0.1895 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4264 | Classification_binary: 3.7257 | Regression_cos_phi: 0.1119 | Regression_energy: 1.9855 | Regression_eta: 0.1158 | Regression_pt: 2.6075 | Regression_sin_phi: 0.1093 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7305s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0817 | max_tensorboard_logging: 0.2164 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2738.9 | uss_MB: 1496.7 | child_rss_MB: 22771.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:34:24,211] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 05:34:24,211] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:04,932] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 05:35:04,932] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:16,271] INFO : Step 26000/500000 rank0 | Train Loss: 8.2615 | Train Loss Unweighted: 3.8360 | Train Loss Components: Classification: 0.0370 | Classification_binary: 3.4348 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1677 | Regression_eta: 0.0068 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4064 | Classification_binary: 3.4348 | Regression_cos_phi: 0.0798 | Regression_energy: 1.8050 | Regression_eta: 0.0542 | Regression_pt: 2.3843 | Regression_sin_phi: 0.0970 | LR: 5.47e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7458s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0253 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.2118 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2739.6 | uss_MB: 1497.5 | child_rss_MB: 22773.4 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:35:16,273] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:35:16,321] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:35:45,711] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 05:35:45,711] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:26,541] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 05:36:26,541] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:28,944] INFO : Step 27000/500000 rank0 | Train Loss: 8.2513 | Train Loss Unweighted: 3.8223 | Train Loss Components: Classification: 0.0371 | Classification_binary: 3.4204 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1645 | Regression_eta: 0.0109 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4081 | Classification_binary: 3.4204 | Regression_cos_phi: 0.0726 | Regression_energy: 1.7699 | Regression_eta: 0.0865 | Regression_pt: 2.4238 | Regression_sin_phi: 0.0701 | LR: 5.74e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7154s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0271 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.2080 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.2 | uss_MB: 1501.1 | child_rss_MB: 22772.6 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:37:08,739] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 05:37:08,739] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:43,253] INFO : Step 28000/500000 rank0 | Train Loss: 8.2893 | Train Loss Unweighted: 3.9938 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.6047 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1644 | Regression_eta: 0.0078 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3465 | Classification_binary: 3.6047 | Regression_cos_phi: 0.0979 | Regression_energy: 1.7695 | Regression_eta: 0.0616 | Regression_pt: 2.3445 | Regression_sin_phi: 0.0646 | LR: 6.01e-05 | DataLoad Time: 0.0054s | Model Forward Time: 0.7079s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0800 | max_tensorboard_logging: 0.2077 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.8 | uss_MB: 1501.6 | child_rss_MB: 22772.9 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:37:43,255] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:37:43,301] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:37:50,626] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 05:37:50,626] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:38:31,736] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 05:38:31,736] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:38:57,121] INFO : Step 29000/500000 rank0 | Train Loss: 9.3294 | Train Loss Unweighted: 4.2131 | Train Loss Components: Classification: 0.0416 | Classification_binary: 3.7484 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1960 | Regression_eta: 0.0092 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4568 | Classification_binary: 3.7484 | Regression_cos_phi: 0.0897 | Regression_energy: 2.1095 | Regression_eta: 0.0733 | Regression_pt: 2.7343 | Regression_sin_phi: 0.1174 | LR: 6.29e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7115s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0278 | max_optimizer_step: 0.0793 | max_tensorboard_logging: 0.2095 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2744.1 | uss_MB: 1501.9 | child_rss_MB: 22773.2 | allocated_MB: 126.1 | reserved_MB: 656.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:39:13,451] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 05:39:13,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:54,550] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 05:39:54,550] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:40:11,233] INFO : Step 30000/500000 rank0 | Train Loss: 8.0436 | Train Loss Unweighted: 3.9024 | Train Loss Components: Classification: 0.0356 | Classification_binary: 3.5233 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1568 | Regression_eta: 0.0128 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3911 | Classification_binary: 3.5233 | Regression_cos_phi: 0.1031 | Regression_energy: 1.6877 | Regression_eta: 0.1015 | Regression_pt: 2.1471 | Regression_sin_phi: 0.0898 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7631s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.1217 | max_tensorboard_logging: 0.2074 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2740.0 | uss_MB: 1497.8 | child_rss_MB: 22772.0 | allocated_MB: 126.1 | reserved_MB: 656.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:40:11,234] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:40:11,282] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:40:11,283] INFO : Running validation on rank0 +rank0 [2026-11-14 05:40:11,283] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:40:11,283] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:40:11,390] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 05:40:11,472] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:40:11,550] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:40:11,596] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:40:11,642] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:40:11,687] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:40:11,733] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:40:11,778] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:40:11,855] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:40:11,932] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:40:12,009] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:40:12,055] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:40:12,100] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:40:12,146] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:40:12,191] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:40:12,237] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:40:12,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:40:12,309] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:40:12,311] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:40:12,314] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:40:12,314] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:40:12,316] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:40:12,319] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:40:12,320] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:40:12,321] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:40:12,324] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:40:12,325] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:40:12,327] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:40:12,327] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:40:12,330] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:40:12,331] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:40:12,332] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:40:13,313] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.0436 | Valid Loss=9.5466 | Train Loss Unweighted=3.9024 | Valid Loss Unweighted=4.1642 | Stale=0 | ETA=580.5m +rank0 [2026-11-14 05:40:13,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:40:13,409] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:40:13,428] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:40:13,487] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:40:14,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:40:14,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:40:14,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:40:14,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:40:14,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:40:14,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:40:14,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:40:15,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:40:15,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:40:15,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:40:15,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:40:15,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:40:15,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:40:16,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:40:16,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:40:16,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:40:16,529] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:40:16,543] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:40:16,930] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:40:17,170] INFO : Plotted number of elements +rank0 [2026-11-14 05:40:18,098] INFO : Plotted jets +rank0 [2026-11-14 05:40:18,792] INFO : Plotted jet ratio +rank0 [2026-11-14 05:40:44,855] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 05:40:44,855] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:26,108] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 05:41:26,108] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:33,371] INFO : Step 31000/500000 rank0 | Train Loss: 7.6375 | Train Loss Unweighted: 3.7111 | Train Loss Components: Classification: 0.0303 | Classification_binary: 3.3545 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1497 | Regression_eta: 0.0073 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3325 | Classification_binary: 3.3545 | Regression_cos_phi: 0.0546 | Regression_energy: 1.6108 | Regression_eta: 0.0576 | Regression_pt: 2.1030 | Regression_sin_phi: 0.1246 | LR: 6.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7437s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0334 | max_optimizer_step: 0.0828 | max_tensorboard_logging: 0.2184 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2864.8 | uss_MB: 1622.6 | child_rss_MB: 22770.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:42:08,014] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 05:42:08,014] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:42:47,454] INFO : Step 32000/500000 rank0 | Train Loss: 8.9294 | Train Loss Unweighted: 4.0396 | Train Loss Components: Classification: 0.0341 | Classification_binary: 3.5952 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1938 | Regression_eta: 0.0106 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3745 | Classification_binary: 3.5952 | Regression_cos_phi: 0.0607 | Regression_energy: 2.0858 | Regression_eta: 0.0842 | Regression_pt: 2.6374 | Regression_sin_phi: 0.0916 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7456s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0022 | max_backward: 0.0332 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.2161 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2864.8 | uss_MB: 1622.6 | child_rss_MB: 22770.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:42:47,456] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:42:47,504] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:42:50,015] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 05:42:50,015] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:43:31,534] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 05:43:31,534] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:01,612] INFO : Step 33000/500000 rank0 | Train Loss: 8.5106 | Train Loss Unweighted: 4.1768 | Train Loss Components: Classification: 0.0351 | Classification_binary: 3.7816 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1631 | Regression_eta: 0.0109 | Regression_pt: 0.1660 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3860 | Classification_binary: 3.7816 | Regression_cos_phi: 0.1066 | Regression_energy: 1.7546 | Regression_eta: 0.0865 | Regression_pt: 2.2842 | Regression_sin_phi: 0.1111 | LR: 7.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7309s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0795 | max_tensorboard_logging: 0.2218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.5 | uss_MB: 1522.3 | child_rss_MB: 22771.0 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 05:44:13,142] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:44:13,142] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:54,071] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:44:54,071] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:15,191] INFO : Step 34000/500000 rank0 | Train Loss: 9.7737 | Train Loss Unweighted: 4.2839 | Train Loss Components: Classification: 0.0352 | Classification_binary: 3.7838 | Regression_cos_phi: 0.0104 | Regression_energy: 0.2133 | Regression_eta: 0.0147 | Regression_pt: 0.2135 | Regression_sin_phi: 0.0130 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3867 | Classification_binary: 3.7838 | Regression_cos_phi: 0.1135 | Regression_energy: 2.2959 | Regression_eta: 0.1169 | Regression_pt: 2.9372 | Regression_sin_phi: 0.1397 | LR: 7.83e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7158s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0269 | max_optimizer_step: 0.0793 | max_tensorboard_logging: 0.2104 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2765.0 | uss_MB: 1522.8 | child_rss_MB: 22770.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:45:15,193] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:45:15,241] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:45:35,792] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:45:35,793] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:16,984] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:46:16,985] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:29,097] INFO : Step 35000/500000 rank0 | Train Loss: 8.2933 | Train Loss Unweighted: 4.0214 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.6335 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1719 | Regression_eta: 0.0081 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3107 | Classification_binary: 3.6335 | Regression_cos_phi: 0.0708 | Regression_energy: 1.8498 | Regression_eta: 0.0645 | Regression_pt: 2.3042 | Regression_sin_phi: 0.0597 | LR: 8.16e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7135s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0281 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2094 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2769.8 | uss_MB: 1527.6 | child_rss_MB: 22771.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:46:58,241] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:46:58,241] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:47:39,152] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:47:39,152] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:47:42,136] INFO : Step 36000/500000 rank0 | Train Loss: 7.9417 | Train Loss Unweighted: 3.6807 | Train Loss Components: Classification: 0.0360 | Classification_binary: 3.2924 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1595 | Regression_eta: 0.0091 | Regression_pt: 0.1625 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3957 | Classification_binary: 3.2924 | Regression_cos_phi: 0.1200 | Regression_energy: 1.7165 | Regression_eta: 0.0723 | Regression_pt: 2.2354 | Regression_sin_phi: 0.1093 | LR: 8.50e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7163s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2102 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2769.8 | uss_MB: 1527.7 | child_rss_MB: 22771.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:47:42,137] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:47:42,184] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:48:20,912] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:48:20,912] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:55,195] INFO : Step 37000/500000 rank0 | Train Loss: 8.1863 | Train Loss Unweighted: 3.7778 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.3774 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1694 | Regression_eta: 0.0081 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3119 | Classification_binary: 3.3774 | Regression_cos_phi: 0.1156 | Regression_energy: 1.8233 | Regression_eta: 0.0645 | Regression_pt: 2.3660 | Regression_sin_phi: 0.1276 | LR: 8.85e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7507s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2426 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2769.9 | uss_MB: 1527.8 | child_rss_MB: 22772.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:49:01,681] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:49:01,681] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:49:41,833] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:49:41,833] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:06,986] INFO : Step 38000/500000 rank0 | Train Loss: 9.1903 | Train Loss Unweighted: 4.1422 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.6797 | Regression_cos_phi: 0.0196 | Regression_energy: 0.1908 | Regression_eta: 0.0154 | Regression_pt: 0.1890 | Regression_sin_phi: 0.0137 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3729 | Classification_binary: 3.6797 | Regression_cos_phi: 0.2140 | Regression_energy: 2.0536 | Regression_eta: 0.1223 | Regression_pt: 2.6004 | Regression_sin_phi: 0.1474 | LR: 9.21e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7058s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0022 | max_backward: 0.0256 | max_optimizer_step: 0.0787 | max_tensorboard_logging: 0.2086 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2765.5 | uss_MB: 1523.4 | child_rss_MB: 22773.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:50:06,987] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:50:07,034] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:50:22,296] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:50:22,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:03,256] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:51:03,257] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:20,260] INFO : Step 39000/500000 rank0 | Train Loss: 8.2477 | Train Loss Unweighted: 3.7351 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.3249 | Regression_cos_phi: 0.0132 | Regression_energy: 0.1714 | Regression_eta: 0.0092 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3052 | Classification_binary: 3.3249 | Regression_cos_phi: 0.1443 | Regression_energy: 1.8441 | Regression_eta: 0.0730 | Regression_pt: 2.4170 | Regression_sin_phi: 0.1391 | LR: 9.57e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7098s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2083 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2770.4 | uss_MB: 1528.2 | child_rss_MB: 22772.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:51:45,182] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:51:45,182] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:26,070] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:52:26,070] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:34,191] INFO : Step 40000/500000 rank0 | Train Loss: 8.4592 | Train Loss Unweighted: 3.7702 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.3415 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1854 | Regression_eta: 0.0127 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3771 | Classification_binary: 3.3415 | Regression_cos_phi: 0.1011 | Regression_energy: 1.9954 | Regression_eta: 0.1008 | Regression_pt: 2.4395 | Regression_sin_phi: 0.1038 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0022 | max_backward: 0.0257 | max_optimizer_step: 0.0804 | max_tensorboard_logging: 0.2074 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2770.7 | uss_MB: 1528.6 | child_rss_MB: 22772.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:52:34,192] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:52:34,239] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:52:34,240] INFO : Running validation on rank0 +rank0 [2026-11-14 05:52:34,240] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:52:34,240] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:52:34,348] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:52:34,429] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:52:34,507] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:52:34,555] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:52:34,601] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:52:34,647] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:52:34,693] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:52:34,739] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:52:34,804] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:52:34,881] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:52:34,957] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:52:35,004] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:52:35,050] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:52:35,095] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:52:35,141] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:52:35,186] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:52:35,252] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:52:35,257] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:52:35,258] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:52:35,260] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:52:35,262] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:52:35,263] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:52:35,265] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:52:35,266] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:52:35,268] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:52:35,269] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:52:35,271] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:52:35,272] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:52:35,274] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:52:35,275] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:52:35,277] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:52:35,278] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:52:36,306] INFO : VALIDATION | Step=40000/500000 | Train Loss=8.4592 | Valid Loss=9.1452 | Train Loss Unweighted=3.7702 | Valid Loss Unweighted=3.9775 | Stale=0 | ETA=568.5m +rank0 [2026-11-14 05:52:36,306] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:52:36,398] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:52:36,406] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:36,453] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:52:36,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:52:37,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:52:37,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:52:37,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:52:37,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:52:37,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:52:37,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:52:38,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:52:38,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:52:38,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:52:38,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:52:38,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:52:38,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:52:38,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:52:39,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:52:39,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:52:39,353] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 05:52:39,359] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:39,747] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:39,997] INFO : Plotted number of elements +rank0 [2026-11-14 05:52:40,889] INFO : Plotted jets +rank0 [2026-11-14 05:52:41,585] INFO : Plotted jet ratio +rank0 [2026-11-14 05:53:15,196] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:53:15,196] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:53:54,562] INFO : Step 41000/500000 rank0 | Train Loss: 7.7864 | Train Loss Unweighted: 3.7823 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.4175 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1520 | Regression_eta: 0.0095 | Regression_pt: 0.1540 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3207 | Classification_binary: 3.4175 | Regression_cos_phi: 0.1139 | Regression_energy: 1.6356 | Regression_eta: 0.0751 | Regression_pt: 2.1189 | Regression_sin_phi: 0.1047 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0798 | max_tensorboard_logging: 0.2116 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2789.5 | uss_MB: 1547.4 | child_rss_MB: 22773.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:53:56,466] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:53:56,466] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:37,472] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:54:37,472] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:55:08,212] INFO : Step 42000/500000 rank0 | Train Loss: 9.1765 | Train Loss Unweighted: 4.1269 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.6690 | Regression_cos_phi: 0.0122 | Regression_energy: 0.1873 | Regression_eta: 0.0123 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0156 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3093 | Classification_binary: 3.6690 | Regression_cos_phi: 0.1334 | Regression_energy: 2.0152 | Regression_eta: 0.0973 | Regression_pt: 2.7838 | Regression_sin_phi: 0.1684 | LR: 1.07e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7094s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2063 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2790.3 | uss_MB: 1548.2 | child_rss_MB: 22774.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:55:08,214] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:55:08,262] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:55:19,041] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:55:19,042] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:00,370] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:56:00,371] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:21,677] INFO : Step 43000/500000 rank0 | Train Loss: 7.5563 | Train Loss Unweighted: 3.4113 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.0382 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1605 | Regression_eta: 0.0063 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2649 | Classification_binary: 3.0382 | Regression_cos_phi: 0.0516 | Regression_energy: 1.7274 | Regression_eta: 0.0497 | Regression_pt: 2.3640 | Regression_sin_phi: 0.0605 | LR: 1.11e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7056s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0802 | max_tensorboard_logging: 0.2020 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.5 | uss_MB: 1552.4 | child_rss_MB: 22774.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:56:41,522] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:56:41,522] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:22,717] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:57:22,717] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:35,463] INFO : Step 44000/500000 rank0 | Train Loss: 7.1248 | Train Loss Unweighted: 3.4840 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.1502 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1382 | Regression_eta: 0.0128 | Regression_pt: 0.1372 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3449 | Classification_binary: 3.1502 | Regression_cos_phi: 0.0645 | Regression_energy: 1.4873 | Regression_eta: 0.1016 | Regression_pt: 1.8870 | Regression_sin_phi: 0.0891 | LR: 1.15e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7186s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.2137 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.9 | uss_MB: 1552.8 | child_rss_MB: 22773.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:57:35,465] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:57:35,512] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:58:04,395] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:58:04,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:45,792] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:58:45,793] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:49,744] INFO : Step 45000/500000 rank0 | Train Loss: 7.9170 | Train Loss Unweighted: 3.7282 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.3424 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1652 | Regression_eta: 0.0150 | Regression_pt: 0.1525 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3461 | Classification_binary: 3.3424 | Regression_cos_phi: 0.1193 | Regression_energy: 1.7777 | Regression_eta: 0.1189 | Regression_pt: 2.0980 | Regression_sin_phi: 0.1145 | LR: 1.19e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7042s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2001 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.9 | uss_MB: 1552.8 | child_rss_MB: 22775.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:59:28,203] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:59:28,203] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:04,016] INFO : Step 46000/500000 rank0 | Train Loss: 7.5449 | Train Loss Unweighted: 3.8439 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.5056 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1384 | Regression_eta: 0.0110 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3143 | Classification_binary: 3.5056 | Regression_cos_phi: 0.0974 | Regression_energy: 1.4897 | Regression_eta: 0.0872 | Regression_pt: 1.9356 | Regression_sin_phi: 0.1151 | LR: 1.23e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7117s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0820 | max_tensorboard_logging: 0.2001 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2790.4 | uss_MB: 1548.3 | child_rss_MB: 22773.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:00:04,018] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 06:00:04,066] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:00:10,292] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 06:00:10,292] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:51,772] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 06:00:51,772] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:18,399] INFO : Step 47000/500000 rank0 | Train Loss: 7.2733 | Train Loss Unweighted: 3.5060 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.1627 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1432 | Regression_eta: 0.0103 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3204 | Classification_binary: 3.1627 | Regression_cos_phi: 0.0677 | Regression_energy: 1.5404 | Regression_eta: 0.0816 | Regression_pt: 2.0106 | Regression_sin_phi: 0.0898 | LR: 1.27e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7085s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0806 | max_tensorboard_logging: 0.2059 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.1 | uss_MB: 1552.0 | child_rss_MB: 22774.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:01:33,421] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 06:01:33,421] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:14,157] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 06:02:14,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:31,739] INFO : Step 48000/500000 rank0 | Train Loss: 7.1753 | Train Loss Unweighted: 3.5857 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.2580 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1295 | Regression_eta: 0.0118 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2908 | Classification_binary: 3.2580 | Regression_cos_phi: 0.1085 | Regression_energy: 1.3930 | Regression_eta: 0.0937 | Regression_pt: 1.9152 | Regression_sin_phi: 0.1161 | LR: 1.31e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7172s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0306 | max_loss: 0.0021 | max_backward: 0.0271 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2061 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2794.2 | uss_MB: 1552.1 | child_rss_MB: 22774.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:02:31,741] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:02:31,787] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:02:56,240] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 06:02:56,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:36,654] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 06:03:36,654] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:44,938] INFO : Step 49000/500000 rank0 | Train Loss: 7.3959 | Train Loss Unweighted: 3.8290 | Train Loss Components: Classification: 0.0245 | Classification_binary: 3.5047 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1386 | Regression_eta: 0.0067 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2697 | Classification_binary: 3.5047 | Regression_cos_phi: 0.0880 | Regression_energy: 1.4912 | Regression_eta: 0.0534 | Regression_pt: 1.9000 | Regression_sin_phi: 0.0888 | LR: 1.36e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7146s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.2144 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.2 | uss_MB: 1552.1 | child_rss_MB: 22775.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:04:18,092] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 06:04:18,092] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:59,090] INFO : Step 50000/500000 rank0 | Train Loss: 7.8626 | Train Loss Unweighted: 3.5490 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.1553 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1626 | Regression_eta: 0.0160 | Regression_pt: 0.1696 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3308 | Classification_binary: 3.1553 | Regression_cos_phi: 0.0825 | Regression_energy: 1.7502 | Regression_eta: 0.1270 | Regression_pt: 2.3335 | Regression_sin_phi: 0.0833 | LR: 1.40e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0786 | max_tensorboard_logging: 0.2141 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2789.7 | uss_MB: 1547.6 | child_rss_MB: 22773.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:04:59,092] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:04:59,139] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:04:59,139] INFO : Running validation on rank0 +rank0 [2026-11-14 06:04:59,140] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:04:59,140] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:04:59,246] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 06:04:59,326] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:04:59,404] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:04:59,454] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:04:59,499] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:04:59,544] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:04:59,590] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:04:59,636] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:04:59,702] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:04:59,779] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:04:59,857] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:04:59,904] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:04:59,950] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:04:59,995] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:05:00,043] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:05:00,089] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:05:00,155] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:05:00,159] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:05:00,161] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:05:00,162] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:05:00,164] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:05:00,165] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:05:00,167] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:05:00,169] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:05:00,170] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:05:00,172] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:05:00,173] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:05:00,175] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:05:00,177] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:05:00,178] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:05:00,180] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:05:00,181] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:05:01,160] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.8626 | Valid Loss=9.0423 | Train Loss Unweighted=3.5490 | Valid Loss Unweighted=3.8565 | Stale=0 | ETA=556.7m +rank0 [2026-11-14 06:05:01,160] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:05:01,264] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:05:01,272] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:05:01,339] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:05:01,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:05:02,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:05:02,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:05:02,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:05:02,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:05:02,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:05:02,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:05:02,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:05:03,142] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:05:03,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:05:03,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:05:03,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:05:03,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:05:03,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:05:04,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:05:04,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:05:04,393] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:05:04,399] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:05:04,771] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:05:05,015] INFO : Plotted number of elements +rank0 [2026-11-14 06:05:05,937] INFO : Plotted jets +rank0 [2026-11-14 06:05:06,632] INFO : Plotted jet ratio +rank0 [2026-11-14 06:05:08,400] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 06:05:08,400] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:49,887] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 06:05:49,887] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:21,388] INFO : Step 51000/500000 rank0 | Train Loss: 7.0579 | Train Loss Unweighted: 3.4916 | Train Loss Components: Classification: 0.0281 | Classification_binary: 3.1659 | Regression_cos_phi: 0.0136 | Regression_energy: 0.1340 | Regression_eta: 0.0078 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3090 | Classification_binary: 3.1659 | Regression_cos_phi: 0.1487 | Regression_energy: 1.4415 | Regression_eta: 0.0619 | Regression_pt: 1.8401 | Regression_sin_phi: 0.0909 | LR: 1.44e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7457s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0300 | max_loss: 0.0021 | max_backward: 0.0270 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.2096 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2923.6 | uss_MB: 1681.5 | child_rss_MB: 22773.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:31,894] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 06:06:31,894] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:13,297] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 06:07:13,297] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:35,677] INFO : Step 52000/500000 rank0 | Train Loss: 7.3492 | Train Loss Unweighted: 3.5153 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.1663 | Regression_cos_phi: 0.0157 | Regression_energy: 0.1492 | Regression_eta: 0.0073 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2721 | Classification_binary: 3.1663 | Regression_cos_phi: 0.1720 | Regression_energy: 1.6054 | Regression_eta: 0.0580 | Regression_pt: 2.0206 | Regression_sin_phi: 0.0547 | LR: 1.49e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7358s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0813 | max_tensorboard_logging: 0.2201 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2923.6 | uss_MB: 1681.5 | child_rss_MB: 22774.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:07:35,678] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 06:07:35,727] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 06:07:55,350] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 06:07:55,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:37,000] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 06:08:37,000] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:50,327] INFO : Step 53000/500000 rank0 | Train Loss: 6.8211 | Train Loss Unweighted: 3.4596 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.1521 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1242 | Regression_eta: 0.0087 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3295 | Classification_binary: 3.1521 | Regression_cos_phi: 0.1147 | Regression_energy: 1.3362 | Regression_eta: 0.0688 | Regression_pt: 1.7277 | Regression_sin_phi: 0.0921 | LR: 1.53e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7357s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2154 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2923.6 | uss_MB: 1681.5 | child_rss_MB: 22771.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:09:19,049] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 06:09:19,049] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:00,285] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 06:10:00,286] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:04,467] INFO : Step 54000/500000 rank0 | Train Loss: 6.3049 | Train Loss Unweighted: 3.2581 | Train Loss Components: Classification: 0.0284 | Classification_binary: 2.9809 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1139 | Regression_eta: 0.0062 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3122 | Classification_binary: 2.9809 | Regression_cos_phi: 0.0659 | Regression_energy: 1.2254 | Regression_eta: 0.0493 | Regression_pt: 1.6128 | Regression_sin_phi: 0.0582 | LR: 1.58e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0820 | max_tensorboard_logging: 0.2161 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2814.9 | uss_MB: 1572.8 | child_rss_MB: 22774.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:10:04,468] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 06:10:04,515] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:10:41,333] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 06:10:41,333] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:17,719] INFO : Step 55000/500000 rank0 | Train Loss: 6.1466 | Train Loss Unweighted: 3.3085 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0494 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1077 | Regression_eta: 0.0068 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2415 | Classification_binary: 3.0494 | Regression_cos_phi: 0.0795 | Regression_energy: 1.1592 | Regression_eta: 0.0537 | Regression_pt: 1.4763 | Regression_sin_phi: 0.0870 | LR: 1.62e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7342s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2123 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.0 | uss_MB: 1574.9 | child_rss_MB: 22775.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:24,029] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 06:11:24,053] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:12:06,856] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 06:12:06,856] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:12:34,262] INFO : Step 56000/500000 rank0 | Train Loss: 5.9982 | Train Loss Unweighted: 3.1164 | Train Loss Components: Classification: 0.0302 | Classification_binary: 2.8507 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1056 | Regression_eta: 0.0099 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3319 | Classification_binary: 2.8507 | Regression_cos_phi: 0.0803 | Regression_energy: 1.1369 | Regression_eta: 0.0783 | Regression_pt: 1.4189 | Regression_sin_phi: 0.1012 | LR: 1.67e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7456s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0333 | max_optimizer_step: 0.0806 | max_tensorboard_logging: 0.2305 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.1 | uss_MB: 1574.9 | child_rss_MB: 22775.6 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:12:34,264] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 06:12:34,339] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:12:49,105] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 06:12:49,105] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:30,708] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 06:13:30,708] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:48,983] INFO : Step 57000/500000 rank0 | Train Loss: 5.7304 | Train Loss Unweighted: 3.0672 | Train Loss Components: Classification: 0.0266 | Classification_binary: 2.8237 | Regression_cos_phi: 0.0071 | Regression_energy: 0.0991 | Regression_eta: 0.0051 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2920 | Classification_binary: 2.8237 | Regression_cos_phi: 0.0773 | Regression_energy: 1.0667 | Regression_eta: 0.0403 | Regression_pt: 1.3460 | Regression_sin_phi: 0.0846 | LR: 1.72e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7106s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0270 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2080 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2818.1 | uss_MB: 1576.0 | child_rss_MB: 22774.6 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:13,006] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 06:14:13,007] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:14:54,940] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 06:14:54,940] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:03,995] INFO : Step 58000/500000 rank0 | Train Loss: 6.3357 | Train Loss Unweighted: 3.4307 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.1621 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1058 | Regression_eta: 0.0139 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2656 | Classification_binary: 3.1621 | Regression_cos_phi: 0.0778 | Regression_energy: 1.1381 | Regression_eta: 0.1103 | Regression_pt: 1.4495 | Regression_sin_phi: 0.1323 | LR: 1.76e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7188s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0021 | max_backward: 0.0269 | max_optimizer_step: 0.0826 | max_tensorboard_logging: 0.2125 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2814.2 | uss_MB: 1572.1 | child_rss_MB: 22773.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:15:03,997] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 06:15:04,048] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 06:15:37,160] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 06:15:37,161] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:16:18,603] INFO : Step 59000/500000 rank0 | Train Loss: 6.8693 | Train Loss Unweighted: 3.4078 | Train Loss Components: Classification: 0.0323 | Classification_binary: 3.0910 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1282 | Regression_eta: 0.0098 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3553 | Classification_binary: 3.0910 | Regression_cos_phi: 0.0855 | Regression_energy: 1.3798 | Regression_eta: 0.0773 | Regression_pt: 1.7835 | Regression_sin_phi: 0.0969 | LR: 1.81e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7607s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0816 | max_tensorboard_logging: 0.2108 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.6 | uss_MB: 1575.5 | child_rss_MB: 22772.7 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:16:19,328] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 06:16:19,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:00,823] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 06:17:00,823] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:33,459] INFO : Step 60000/500000 rank0 | Train Loss: 6.1460 | Train Loss Unweighted: 3.2364 | Train Loss Components: Classification: 0.0296 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1051 | Regression_eta: 0.0103 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3249 | Classification_binary: 2.9674 | Regression_cos_phi: 0.1245 | Regression_energy: 1.1310 | Regression_eta: 0.0817 | Regression_pt: 1.4003 | Regression_sin_phi: 0.1162 | LR: 1.86e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7277s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0270 | max_loss: 0.0021 | max_backward: 0.0270 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.2150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.8 | uss_MB: 1575.7 | child_rss_MB: 22771.5 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:17:33,461] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 06:17:33,509] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 06:17:33,511] INFO : Running validation on rank0 +rank0 [2026-11-14 06:17:33,511] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:17:33,511] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:17:33,649] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 06:17:33,732] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:17:33,830] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:17:33,879] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:17:33,924] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:17:33,969] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:17:34,016] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:17:34,062] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:17:34,128] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:17:34,205] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:17:34,282] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:17:34,329] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:17:34,376] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:17:34,423] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:17:34,469] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:17:34,515] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:17:34,583] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:17:34,587] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:17:34,589] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:17:34,591] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:17:34,592] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:17:34,594] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:17:34,596] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:17:34,597] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:17:34,598] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:17:34,600] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:17:34,601] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:17:34,603] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:17:34,604] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:17:34,606] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:17:34,608] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:17:34,609] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:17:35,542] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.1460 | Valid Loss=9.5209 | Train Loss Unweighted=3.2364 | Valid Loss Unweighted=3.9083 | Stale=1 | ETA=545.8m +rank0 [2026-11-14 06:17:35,542] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:17:35,641] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:17:35,650] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:17:35,697] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:17:36,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:17:36,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:17:36,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:17:36,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:17:36,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:17:36,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:17:37,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:17:37,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:17:37,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:17:37,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:17:37,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:17:37,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:17:38,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:17:38,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:17:38,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:17:38,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:17:38,691] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:17:38,698] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:17:39,074] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:17:39,317] INFO : Plotted number of elements +rank0 [2026-11-14 06:17:40,221] INFO : Plotted jets +rank0 [2026-11-14 06:17:40,920] INFO : Plotted jet ratio +rank0 [2026-11-14 06:17:51,318] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 06:17:51,318] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:32,944] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 06:18:32,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:56,071] INFO : Step 61000/500000 rank0 | Train Loss: 6.2206 | Train Loss Unweighted: 3.3921 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.1288 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1023 | Regression_eta: 0.0175 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3058 | Classification_binary: 3.1288 | Regression_cos_phi: 0.0864 | Regression_energy: 1.1013 | Regression_eta: 0.1386 | Regression_pt: 1.3818 | Regression_sin_phi: 0.0779 | LR: 1.91e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7363s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0021 | max_backward: 0.0271 | max_optimizer_step: 0.0801 | max_tensorboard_logging: 0.2124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2944.2 | uss_MB: 1702.1 | child_rss_MB: 22764.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:19:15,118] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 06:19:15,118] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:56,570] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 06:19:56,571] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:10,514] INFO : Step 62000/500000 rank0 | Train Loss: 6.4635 | Train Loss Unweighted: 3.2660 | Train Loss Components: Classification: 0.0279 | Classification_binary: 2.9747 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1219 | Regression_eta: 0.0058 | Regression_pt: 0.1211 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3061 | Classification_binary: 2.9747 | Regression_cos_phi: 0.0933 | Regression_energy: 1.3117 | Regression_eta: 0.0459 | Regression_pt: 1.6663 | Regression_sin_phi: 0.0655 | LR: 1.95e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7370s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0823 | max_tensorboard_logging: 0.2188 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.9 | uss_MB: 1597.8 | child_rss_MB: 22762.8 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:20:10,516] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 06:20:10,569] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 06:20:38,796] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 06:20:38,796] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:20,525] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 06:21:20,525] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:25,262] INFO : Step 63000/500000 rank0 | Train Loss: 5.7807 | Train Loss Unweighted: 3.1847 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.9480 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0944 | Regression_eta: 0.0059 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2716 | Classification_binary: 2.9480 | Regression_cos_phi: 0.0539 | Regression_energy: 1.0161 | Regression_eta: 0.0464 | Regression_pt: 1.3621 | Regression_sin_phi: 0.0825 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7241s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0276 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.2072 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2843.5 | uss_MB: 1601.4 | child_rss_MB: 22763.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:22:02,414] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 06:22:02,414] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:39,267] INFO : Step 64000/500000 rank0 | Train Loss: 5.9483 | Train Loss Unweighted: 2.9071 | Train Loss Components: Classification: 0.0280 | Classification_binary: 2.6271 | Regression_cos_phi: 0.0120 | Regression_energy: 0.1098 | Regression_eta: 0.0090 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3079 | Classification_binary: 2.6271 | Regression_cos_phi: 0.1311 | Regression_energy: 1.1812 | Regression_eta: 0.0709 | Regression_pt: 1.4909 | Regression_sin_phi: 0.1392 | LR: 2.05e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7693s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0805 | max_tensorboard_logging: 0.2067 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2843.6 | uss_MB: 1601.5 | child_rss_MB: 22766.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:22:39,268] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 06:22:39,318] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 06:22:44,177] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 06:22:44,177] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:24,207] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 06:23:24,207] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:52,393] INFO : Step 65000/500000 rank0 | Train Loss: 6.7107 | Train Loss Unweighted: 3.3816 | Train Loss Components: Classification: 0.0334 | Classification_binary: 3.0783 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1214 | Regression_eta: 0.0049 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3665 | Classification_binary: 3.0783 | Regression_cos_phi: 0.1012 | Regression_energy: 1.3061 | Regression_eta: 0.0390 | Regression_pt: 1.7140 | Regression_sin_phi: 0.1056 | LR: 2.10e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0244 | max_loss: 0.0021 | max_backward: 0.0273 | max_optimizer_step: 0.0807 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2844.5 | uss_MB: 1602.4 | child_rss_MB: 22763.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:24:06,523] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 06:24:06,523] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:47,940] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 06:24:47,941] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:06,708] INFO : Step 66000/500000 rank0 | Train Loss: 6.2999 | Train Loss Unweighted: 3.3176 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.0449 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1097 | Regression_eta: 0.0078 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2983 | Classification_binary: 3.0449 | Regression_cos_phi: 0.0833 | Regression_energy: 1.1803 | Regression_eta: 0.0617 | Regression_pt: 1.5430 | Regression_sin_phi: 0.0882 | LR: 2.15e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7158s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0022 | max_backward: 0.0269 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2080 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.9 | uss_MB: 1597.8 | child_rss_MB: 22764.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:25:06,710] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 06:25:06,761] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 06:25:30,057] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 06:25:30,057] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:11,532] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 06:26:11,533] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:21,198] INFO : Step 67000/500000 rank0 | Train Loss: 6.3735 | Train Loss Unweighted: 3.2323 | Train Loss Components: Classification: 0.0272 | Classification_binary: 2.9449 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1197 | Regression_eta: 0.0076 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2985 | Classification_binary: 2.9449 | Regression_cos_phi: 0.1006 | Regression_energy: 1.2886 | Regression_eta: 0.0605 | Regression_pt: 1.6074 | Regression_sin_phi: 0.0729 | LR: 2.20e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7074s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2053 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2843.5 | uss_MB: 1601.4 | child_rss_MB: 22765.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 30.3 +rank0 [2026-11-14 06:26:53,923] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 06:26:53,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:36,105] INFO : Step 68000/500000 rank0 | Train Loss: 6.3533 | Train Loss Unweighted: 3.3041 | Train Loss Components: Classification: 0.0244 | Classification_binary: 3.0251 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1137 | Regression_eta: 0.0084 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2683 | Classification_binary: 3.0251 | Regression_cos_phi: 0.1022 | Regression_energy: 1.2235 | Regression_eta: 0.0665 | Regression_pt: 1.5711 | Regression_sin_phi: 0.0967 | LR: 2.25e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7179s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0035 | max_backward: 0.0267 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.2084 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2843.6 | uss_MB: 1601.5 | child_rss_MB: 22765.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:27:36,107] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 06:27:36,153] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 06:27:36,299] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 06:27:36,299] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:17,682] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 06:28:17,682] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:50,451] INFO : Step 69000/500000 rank0 | Train Loss: 5.4821 | Train Loss Unweighted: 3.0174 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7926 | Regression_cos_phi: 0.0106 | Regression_energy: 0.0920 | Regression_eta: 0.0050 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2030 | Classification_binary: 2.7926 | Regression_cos_phi: 0.1163 | Regression_energy: 0.9899 | Regression_eta: 0.0397 | Regression_pt: 1.2759 | Regression_sin_phi: 0.0648 | LR: 2.30e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7072s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0022 | max_backward: 0.0269 | max_optimizer_step: 0.0834 | max_tensorboard_logging: 0.1943 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2843.8 | uss_MB: 1601.7 | child_rss_MB: 22766.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:28:59,674] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 06:28:59,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:40,971] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 06:29:40,971] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:04,817] INFO : Step 70000/500000 rank0 | Train Loss: 5.4883 | Train Loss Unweighted: 3.1045 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8848 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0907 | Regression_eta: 0.0081 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2230 | Classification_binary: 2.8848 | Regression_cos_phi: 0.0757 | Regression_energy: 0.9764 | Regression_eta: 0.0641 | Regression_pt: 1.1783 | Regression_sin_phi: 0.0861 | LR: 2.35e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0270 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.1949 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2840.0 | uss_MB: 1597.9 | child_rss_MB: 22770.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:30:04,818] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 06:30:04,864] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 06:30:04,865] INFO : Running validation on rank0 +rank0 [2026-11-14 06:30:04,865] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:30:04,865] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:30:04,973] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 06:30:05,054] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:30:05,131] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:30:05,181] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:30:05,227] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:30:05,272] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:30:05,317] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:30:05,364] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:30:05,429] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:30:05,507] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:30:05,584] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:30:05,632] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:30:05,678] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:30:05,723] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:30:05,769] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:30:05,815] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:30:05,879] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:30:05,883] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:30:05,885] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:30:05,886] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:30:05,888] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:30:05,889] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:30:05,892] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:30:05,892] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:30:05,894] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:30:05,896] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:30:05,897] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:30:05,899] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:30:05,900] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:30:05,902] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:30:05,903] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:30:05,905] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:30:06,843] INFO : VALIDATION | Step=70000/500000 | Train Loss=5.4883 | Valid Loss=9.6698 | Train Loss Unweighted=3.1045 | Valid Loss Unweighted=4.0003 | Stale=2 | ETA=534.1m +rank0 [2026-11-14 06:30:06,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:30:06,935] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:30:06,944] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:06,991] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:30:07,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:30:07,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:30:07,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:30:07,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:30:08,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:30:08,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:30:08,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:30:08,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:30:08,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:30:08,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:30:09,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:30:09,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:30:09,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:30:09,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:30:09,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:30:09,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:30:09,942] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:30:09,949] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:10,334] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:10,577] INFO : Plotted number of elements +rank0 [2026-11-14 06:30:11,502] INFO : Plotted jets +rank0 [2026-11-14 06:30:12,196] INFO : Plotted jet ratio +rank0 [2026-11-14 06:30:30,916] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 06:30:30,917] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:11,907] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 06:31:11,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:26,124] INFO : Step 71000/500000 rank0 | Train Loss: 4.7648 | Train Loss Unweighted: 2.7346 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.5480 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0744 | Regression_eta: 0.0064 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2358 | Classification_binary: 2.5480 | Regression_cos_phi: 0.0581 | Regression_energy: 0.8008 | Regression_eta: 0.0510 | Regression_pt: 1.0195 | Regression_sin_phi: 0.0515 | LR: 2.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7218s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0802 | max_tensorboard_logging: 0.2063 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2971.6 | uss_MB: 1729.5 | child_rss_MB: 22770.0 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:31:53,559] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 06:31:53,560] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:34,863] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 06:32:34,863] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:40,241] INFO : Step 72000/500000 rank0 | Train Loss: 5.3242 | Train Loss Unweighted: 2.9626 | Train Loss Components: Classification: 0.0244 | Classification_binary: 2.7462 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0888 | Regression_eta: 0.0064 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2685 | Classification_binary: 2.7462 | Regression_cos_phi: 0.0584 | Regression_energy: 0.9561 | Regression_eta: 0.0506 | Regression_pt: 1.1968 | Regression_sin_phi: 0.0475 | LR: 2.45e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7297s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0268 | max_optimizer_step: 0.0836 | max_tensorboard_logging: 0.2082 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2971.4 | uss_MB: 1729.3 | child_rss_MB: 22771.1 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:32:40,243] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 06:32:40,293] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 06:33:17,131] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 06:33:17,131] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:33:54,796] INFO : Step 73000/500000 rank0 | Train Loss: 5.4729 | Train Loss Unweighted: 3.0059 | Train Loss Components: Classification: 0.0256 | Classification_binary: 2.7792 | Regression_cos_phi: 0.0092 | Regression_energy: 0.0914 | Regression_eta: 0.0066 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2810 | Classification_binary: 2.7792 | Regression_cos_phi: 0.1005 | Regression_energy: 0.9835 | Regression_eta: 0.0524 | Regression_pt: 1.2160 | Regression_sin_phi: 0.0603 | LR: 2.50e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7216s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0831 | max_tensorboard_logging: 0.2056 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2971.4 | uss_MB: 1729.3 | child_rss_MB: 22770.4 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:33:59,206] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 06:33:59,206] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:40,517] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 06:34:40,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:08,965] INFO : Step 74000/500000 rank0 | Train Loss: 5.4256 | Train Loss Unweighted: 2.9539 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7275 | Regression_cos_phi: 0.0092 | Regression_energy: 0.0910 | Regression_eta: 0.0052 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2341 | Classification_binary: 2.7275 | Regression_cos_phi: 0.1002 | Regression_energy: 0.9794 | Regression_eta: 0.0410 | Regression_pt: 1.2425 | Regression_sin_phi: 0.1008 | LR: 2.55e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7185s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0804 | max_tensorboard_logging: 0.2099 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2864.8 | uss_MB: 1622.7 | child_rss_MB: 22770.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:35:08,967] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 06:35:09,017] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 06:35:22,603] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 06:35:22,603] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:04,033] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 06:36:04,034] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:23,852] INFO : Step 75000/500000 rank0 | Train Loss: 5.3375 | Train Loss Unweighted: 3.0206 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.8072 | Regression_cos_phi: 0.0059 | Regression_energy: 0.0836 | Regression_eta: 0.0088 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2629 | Classification_binary: 2.8072 | Regression_cos_phi: 0.0645 | Regression_energy: 0.9000 | Regression_eta: 0.0697 | Regression_pt: 1.1577 | Regression_sin_phi: 0.0755 | LR: 2.60e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7412s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0302 | max_loss: 0.0021 | max_backward: 0.0270 | max_optimizer_step: 0.0826 | max_tensorboard_logging: 0.2077 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2870.6 | uss_MB: 1628.5 | child_rss_MB: 22769.7 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:36:46,436] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 06:36:46,436] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:27,172] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 06:37:27,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:37,228] INFO : Step 76000/500000 rank0 | Train Loss: 4.9390 | Train Loss Unweighted: 2.6609 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.4517 | Regression_cos_phi: 0.0063 | Regression_energy: 0.0839 | Regression_eta: 0.0072 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2405 | Classification_binary: 2.4517 | Regression_cos_phi: 0.0689 | Regression_energy: 0.9030 | Regression_eta: 0.0574 | Regression_pt: 1.1536 | Regression_sin_phi: 0.0638 | LR: 2.65e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7059s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0257 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2050 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2870.8 | uss_MB: 1628.7 | child_rss_MB: 22771.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:37:37,230] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 06:37:37,278] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 06:38:08,255] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 06:38:08,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:49,294] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 06:38:49,294] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:50,420] INFO : Step 77000/500000 rank0 | Train Loss: 4.3600 | Train Loss Unweighted: 2.4897 | Train Loss Components: Classification: 0.0249 | Classification_binary: 2.3173 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0681 | Regression_eta: 0.0060 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2734 | Classification_binary: 2.3173 | Regression_cos_phi: 0.0480 | Regression_energy: 0.7330 | Regression_eta: 0.0473 | Regression_pt: 0.9082 | Regression_sin_phi: 0.0328 | LR: 2.70e-04 | DataLoad Time: 0.0003s | Model Forward Time: 0.7517s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0022 | max_backward: 0.0266 | max_optimizer_step: 0.1088 | max_tensorboard_logging: 0.2081 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2871.0 | uss_MB: 1628.9 | child_rss_MB: 22773.2 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:39:31,606] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 06:39:31,606] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:04,709] INFO : Step 78000/500000 rank0 | Train Loss: 4.2693 | Train Loss Unweighted: 2.4605 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.2943 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0663 | Regression_eta: 0.0049 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2051 | Classification_binary: 2.2943 | Regression_cos_phi: 0.0636 | Regression_energy: 0.7130 | Regression_eta: 0.0389 | Regression_pt: 0.8962 | Regression_sin_phi: 0.0582 | LR: 2.75e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7212s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0268 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2071 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2871.0 | uss_MB: 1628.9 | child_rss_MB: 22772.1 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:40:04,711] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 06:40:04,759] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 06:40:13,396] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 06:40:13,396] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:54,619] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 06:40:54,620] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:41:18,752] INFO : Step 79000/500000 rank0 | Train Loss: 4.4165 | Train Loss Unweighted: 2.5798 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.4109 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0674 | Regression_eta: 0.0048 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2293 | Classification_binary: 2.4109 | Regression_cos_phi: 0.0548 | Regression_energy: 0.7252 | Regression_eta: 0.0383 | Regression_pt: 0.9012 | Regression_sin_phi: 0.0569 | LR: 2.80e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7303s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0327 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.2076 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2870.7 | uss_MB: 1628.6 | child_rss_MB: 22771.6 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:41:36,546] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 06:41:36,546] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:17,933] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 06:42:17,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:33,387] INFO : Step 80000/500000 rank0 | Train Loss: 4.7619 | Train Loss Unweighted: 2.6209 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.4237 | Regression_cos_phi: 0.0057 | Regression_energy: 0.0805 | Regression_eta: 0.0069 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2359 | Classification_binary: 2.4237 | Regression_cos_phi: 0.0626 | Regression_energy: 0.8662 | Regression_eta: 0.0546 | Regression_pt: 1.0558 | Regression_sin_phi: 0.0631 | LR: 2.85e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7247s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0336 | max_optimizer_step: 0.0811 | max_tensorboard_logging: 0.2098 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2871.4 | uss_MB: 1629.3 | child_rss_MB: 22772.5 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:42:33,389] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 06:42:33,435] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 06:42:33,436] INFO : Running validation on rank0 +rank0 [2026-11-14 06:42:33,436] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:42:33,436] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:42:33,543] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 06:42:33,624] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:42:33,701] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:42:33,749] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:42:33,795] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:42:33,840] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:42:33,886] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:42:33,931] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:42:33,999] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:42:34,077] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:42:34,154] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:42:34,200] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:42:34,246] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:42:34,291] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:42:34,337] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:42:34,383] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:42:34,449] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:42:34,454] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:42:34,455] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:42:34,457] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:42:34,458] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:42:34,460] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:42:34,461] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:42:34,463] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:42:34,464] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:42:34,466] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:42:34,468] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:42:34,469] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:42:34,471] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:42:34,472] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:42:34,474] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:42:34,475] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:42:35,408] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.7619 | Valid Loss=10.7437 | Train Loss Unweighted=2.6209 | Valid Loss Unweighted=4.3890 | Stale=3 | ETA=522.0m +rank0 [2026-11-14 06:42:35,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:42:35,503] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:42:35,512] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:42:35,560] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:42:35,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:42:36,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:42:36,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:42:36,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:42:36,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:42:36,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:42:36,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:42:37,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:42:37,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:42:37,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:42:37,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:42:37,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:42:37,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:42:38,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:42:38,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:42:38,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:42:38,491] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:42:38,498] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:42:38,892] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:42:39,136] INFO : Plotted number of elements +rank0 [2026-11-14 06:42:40,054] INFO : Plotted jets +rank0 [2026-11-14 06:42:40,746] INFO : Plotted jet ratio +rank0 [2026-11-14 06:42:41,227] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 06:42:41,227] INFO : Training completed. Total time on device 0: 99.520min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..20f54013a5c1c89d1f94f5dd3d43e8b2923b7ab7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/train.log.0 @@ -0,0 +1,1351 @@ +rank0 [2026-11-14 05:03:09,085] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:03:09,085] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:03:09,085] INFO : Instantiating model +rank0 [2026-11-14 05:03:09,103] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:03:09,103] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:03:09,104] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:03:09,105] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:03:09,105] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:03:09,105] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:03:09,105] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,107] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:03:09,107] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,109] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:03:09,109] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,111] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:03:09,112] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,114] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:03:09,114] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,116] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:03:09,116] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,119] INFO : Backbone initialization took 0.01s +rank0 [2026-11-14 05:03:09,119] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:03:09,119] INFO : Initializing output DNNs +rank0 [2026-11-14 05:03:09,120] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,122] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:03:09,129] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:03:09,129] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:03:09,130] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:03:09,130] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:03:09,130] INFO : task_queries enabled=True +rank0 [2026-11-14 05:03:09,130] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:03:09,130] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:03:09,130] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:03:09,130] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:03:09,130] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:03:09,130] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:03:09,130] INFO : output_mode=elementwise +rank0 [2026-11-14 05:03:09,130] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:03:09,130] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:03:09,130] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:03:09,130] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:03:09,130] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:03:09,130] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:03:09,130] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:03:09,130] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:03:09,131] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:03:09,131] INFO : MLPF __init__ done +rank0 [2026-11-14 05:03:09,131] INFO : Instantiated model +rank0 [2026-11-14 05:03:09,131] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:03:09,137] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:03:09,138] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:03:09,198] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:03:09,198] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:03:09,199] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:03:09,199] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:03:09,199] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:03:09,201] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:03:09,202] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935 +rank0 [2026-11-14 05:03:09,202] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935 +rank0 [2026-11-14 05:03:09,202] INFO : Getting dataloaders +rank0 [2026-11-14 05:03:09,202] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:03:09,361] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,399] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,455] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,492] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,525] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,571] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,613] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,659] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,695] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,746] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:03:09,746] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:03:09,746] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:03:09,746] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:03:09,746] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:03:09,756] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,766] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,775] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,784] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,794] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,803] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,813] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,822] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,832] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,841] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:03:09,841] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:03:09,842] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:03:09,871] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:03:10,021] INFO : Got dataloaders +rank0 [2026-11-14 05:03:10,022] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:03:10,022] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:03:10,022] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:03:10,025] INFO : Creating train_iterator +rank0 [2026-11-14 05:03:10,025] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:03:10,052] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:03:10,199] INFO : Created train_iterator +rank0 [2026-11-14 05:03:18,680] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.150242385864257, 'Classification': 0.6509133404493332, 'Regression_pt': 0.5197731229662895, 'Regression_eta': 0.9022421351075173, 'Regression_sin_phi': 0.6643005856871604, 'Regression_cos_phi': 0.6540828502178192, 'Regression_energy': 0.6644598841667175}, weights={'Classification_binary': 1.0, 'Classification': 10.98493766784668, 'Regression_pt': 13.756467819213867, 'Regression_eta': 7.924970626831055, 'Regression_sin_phi': 10.763565063476562, 'Regression_cos_phi': 10.931707382202148, 'Regression_energy': 10.760984420776367} +rank0 [2026-11-14 05:03:52,570] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:03:52,570] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:24,547] INFO : Step 1000/500000 rank0 | Train Loss: 12.6559 | Train Loss Unweighted: 5.1529 | Train Loss Components: Classification: 0.0637 | Classification_binary: 4.4608 | Regression_cos_phi: 0.0216 | Regression_energy: 0.2681 | Regression_eta: 0.0365 | Regression_pt: 0.2779 | Regression_sin_phi: 0.0244 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.6997 | Classification_binary: 4.4608 | Regression_cos_phi: 0.2358 | Regression_energy: 2.8856 | Regression_eta: 0.2896 | Regression_pt: 3.8224 | Regression_sin_phi: 0.2621 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7175s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0021 | max_backward: 0.0254 | max_optimizer_step: 0.0773 | max_tensorboard_logging: 0.2240 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2601.1 | uss_MB: 1351.0 | child_rss_MB: 22759.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:04:33,597] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:04:33,598] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:14,686] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:05:14,687] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:38,256] INFO : Step 2000/500000 rank0 | Train Loss: 11.3900 | Train Loss Unweighted: 4.8544 | Train Loss Components: Classification: 0.0438 | Classification_binary: 4.2578 | Regression_cos_phi: 0.0157 | Regression_energy: 0.2469 | Regression_eta: 0.0202 | Regression_pt: 0.2528 | Regression_sin_phi: 0.0173 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4811 | Classification_binary: 4.2578 | Regression_cos_phi: 0.1714 | Regression_energy: 2.6565 | Regression_eta: 0.1599 | Regression_pt: 3.4772 | Regression_sin_phi: 0.1861 | LR: 2.02e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7101s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0233 | max_loss: 0.0022 | max_backward: 0.0272 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2603.3 | uss_MB: 1352.4 | child_rss_MB: 22760.9 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:05:38,257] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:05:56,449] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:05:56,449] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:37,847] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:06:37,847] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:52,384] INFO : Step 3000/500000 rank0 | Train Loss: 10.9199 | Train Loss Unweighted: 4.8681 | Train Loss Components: Classification: 0.0405 | Classification_binary: 4.3168 | Regression_cos_phi: 0.0165 | Regression_energy: 0.2271 | Regression_eta: 0.0173 | Regression_pt: 0.2361 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4447 | Classification_binary: 4.3168 | Regression_cos_phi: 0.1803 | Regression_energy: 2.4433 | Regression_eta: 0.1374 | Regression_pt: 3.2477 | Regression_sin_phi: 0.1498 | LR: 2.05e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7104s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0022 | max_backward: 0.0263 | max_optimizer_step: 0.0775 | max_tensorboard_logging: 0.2253 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2606.8 | uss_MB: 1356.1 | child_rss_MB: 22760.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:07:19,567] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:07:19,567] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:00,469] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:08:00,469] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:08:05,820] INFO : Step 4000/500000 rank0 | Train Loss: 9.5930 | Train Loss Unweighted: 4.6482 | Train Loss Components: Classification: 0.0418 | Classification_binary: 4.1931 | Regression_cos_phi: 0.0154 | Regression_energy: 0.1798 | Regression_eta: 0.0207 | Regression_pt: 0.1832 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4590 | Classification_binary: 4.1931 | Regression_cos_phi: 0.1680 | Regression_energy: 1.9349 | Regression_eta: 0.1643 | Regression_pt: 2.5203 | Regression_sin_phi: 0.1535 | LR: 2.08e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.6956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0233 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0758 | max_tensorboard_logging: 0.2126 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2606.8 | uss_MB: 1356.2 | child_rss_MB: 22764.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:08:05,822] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:08:41,286] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:08:41,286] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:18,490] INFO : Step 5000/500000 rank0 | Train Loss: 9.1201 | Train Loss Unweighted: 4.4488 | Train Loss Components: Classification: 0.0403 | Classification_binary: 4.0176 | Regression_cos_phi: 0.0154 | Regression_energy: 0.1712 | Regression_eta: 0.0192 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0164 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4429 | Classification_binary: 4.0176 | Regression_cos_phi: 0.1679 | Regression_energy: 1.8417 | Regression_eta: 0.1519 | Regression_pt: 2.3218 | Regression_sin_phi: 0.1763 | LR: 2.13e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7067s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2104 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2635.0 | uss_MB: 1384.3 | child_rss_MB: 22763.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:09:22,692] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:09:22,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:03,423] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:10:03,423] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:31,202] INFO : Step 6000/500000 rank0 | Train Loss: 9.5896 | Train Loss Unweighted: 4.3296 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.8537 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2027 | Regression_eta: 0.0121 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3468 | Classification_binary: 3.8537 | Regression_cos_phi: 0.0885 | Regression_energy: 2.1808 | Regression_eta: 0.0957 | Regression_pt: 2.9437 | Regression_sin_phi: 0.0803 | LR: 2.19e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7075s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0770 | max_tensorboard_logging: 0.2187 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2635.0 | uss_MB: 1384.4 | child_rss_MB: 22765.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:10:31,204] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:10:44,376] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:10:44,376] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:25,345] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:11:25,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:44,726] INFO : Step 7000/500000 rank0 | Train Loss: 9.2235 | Train Loss Unweighted: 4.1558 | Train Loss Components: Classification: 0.0354 | Classification_binary: 3.6979 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1947 | Regression_eta: 0.0078 | Regression_pt: 0.2041 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3890 | Classification_binary: 3.6979 | Regression_cos_phi: 0.0832 | Regression_energy: 2.0953 | Regression_eta: 0.0615 | Regression_pt: 2.8077 | Regression_sin_phi: 0.0888 | LR: 2.26e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6980s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0769 | max_tensorboard_logging: 0.2091 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2635.5 | uss_MB: 1384.9 | child_rss_MB: 22764.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:12:06,967] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:12:06,967] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:48,285] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:12:48,285] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:58,444] INFO : Step 8000/500000 rank0 | Train Loss: 9.2284 | Train Loss Unweighted: 4.3518 | Train Loss Components: Classification: 0.0378 | Classification_binary: 3.9086 | Regression_cos_phi: 0.0158 | Regression_energy: 0.1838 | Regression_eta: 0.0073 | Regression_pt: 0.1868 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4155 | Classification_binary: 3.9086 | Regression_cos_phi: 0.1725 | Regression_energy: 1.9777 | Regression_eta: 0.0582 | Regression_pt: 2.5694 | Regression_sin_phi: 0.1265 | LR: 2.34e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7025s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0025 | max_backward: 0.0260 | max_optimizer_step: 0.0778 | max_tensorboard_logging: 0.2096 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2635.5 | uss_MB: 1384.9 | child_rss_MB: 22764.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:12:58,446] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:12:58,499] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:13:29,977] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:13:29,977] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:14:10,886] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:14:10,886] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:14:12,121] INFO : Step 9000/500000 rank0 | Train Loss: 10.6784 | Train Loss Unweighted: 4.7343 | Train Loss Components: Classification: 0.0450 | Classification_binary: 4.1951 | Regression_cos_phi: 0.0104 | Regression_energy: 0.2275 | Regression_eta: 0.0117 | Regression_pt: 0.2342 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4947 | Classification_binary: 4.1951 | Regression_cos_phi: 0.1141 | Regression_energy: 2.4477 | Regression_eta: 0.0926 | Regression_pt: 3.2218 | Regression_sin_phi: 0.1125 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2208 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2604.8 | uss_MB: 1354.1 | child_rss_MB: 22766.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 22.3 +rank0 [2026-11-14 05:14:52,270] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:14:52,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:25,476] INFO : Step 10000/500000 rank0 | Train Loss: 8.6279 | Train Loss Unweighted: 4.0737 | Train Loss Components: Classification: 0.0379 | Classification_binary: 3.6564 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1739 | Regression_eta: 0.0143 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4162 | Classification_binary: 3.6564 | Regression_cos_phi: 0.1262 | Regression_energy: 1.8710 | Regression_eta: 0.1130 | Regression_pt: 2.3442 | Regression_sin_phi: 0.1009 | LR: 2.52e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7042s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0020 | max_backward: 0.0254 | max_optimizer_step: 0.0771 | max_tensorboard_logging: 0.2119 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2605.3 | uss_MB: 1354.7 | child_rss_MB: 22766.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:15:25,477] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:15:25,523] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:15:25,524] INFO : Running validation on rank0 +rank0 [2026-11-14 05:15:25,524] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:15:25,524] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:15:25,647] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:15:25,765] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:15:25,821] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:15:25,861] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:15:25,903] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:15:25,936] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:15:25,975] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:15:26,015] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:15:26,051] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:15:26,105] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:15:26,159] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:15:26,197] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:15:26,236] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:15:26,269] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:15:26,308] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:15:26,347] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:15:26,383] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:15:26,386] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:15:26,387] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:15:26,389] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:15:26,390] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:15:26,392] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:15:26,394] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:15:26,395] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:15:26,397] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:15:26,398] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:15:26,400] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:15:26,401] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:15:26,403] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:15:26,404] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:15:26,406] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:15:26,408] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:15:27,391] INFO : VALIDATION | Step=10000/500000 | Train Loss=8.6279 | Valid Loss=10.3918 | Train Loss Unweighted=4.0737 | Valid Loss Unweighted=4.4945 | Stale=0 | ETA=602.2m +rank0 [2026-11-14 05:15:27,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:15:27,488] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:15:27,496] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:15:27,539] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:15:28,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:15:28,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:15:28,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:15:28,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:15:28,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:15:28,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:15:28,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:15:29,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:15:29,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:15:29,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:15:29,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:15:29,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:15:29,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:15:30,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:15:30,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:15:30,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:15:30,504] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:15:30,512] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:15:30,894] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:15:31,199] INFO : Plotted number of elements +rank0 [2026-11-14 05:15:32,156] INFO : Plotted jets +rank0 [2026-11-14 05:15:32,840] INFO : Plotted jet ratio +rank0 [2026-11-14 05:15:41,484] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:15:41,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:21,314] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:16:21,315] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:45,127] INFO : Step 11000/500000 rank0 | Train Loss: 9.5722 | Train Loss Unweighted: 4.1320 | Train Loss Components: Classification: 0.0387 | Classification_binary: 3.6402 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2130 | Regression_eta: 0.0062 | Regression_pt: 0.2159 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4246 | Classification_binary: 3.6402 | Regression_cos_phi: 0.1183 | Regression_energy: 2.2921 | Regression_eta: 0.0489 | Regression_pt: 2.9705 | Regression_sin_phi: 0.0775 | LR: 2.63e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7629s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.2202 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.4 | uss_MB: 1522.1 | child_rss_MB: 22767.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:17:02,665] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:17:02,665] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:43,724] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:17:43,724] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:58,923] INFO : Step 12000/500000 rank0 | Train Loss: 9.1185 | Train Loss Unweighted: 4.2970 | Train Loss Components: Classification: 0.0389 | Classification_binary: 3.8553 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1769 | Regression_eta: 0.0156 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0185 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4274 | Classification_binary: 3.8553 | Regression_cos_phi: 0.1142 | Regression_energy: 1.9035 | Regression_eta: 0.1237 | Regression_pt: 2.4949 | Regression_sin_phi: 0.1995 | LR: 2.75e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7159s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2161 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.4 | uss_MB: 1522.2 | child_rss_MB: 22769.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:17:58,925] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:17:58,975] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:18:25,322] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:18:25,322] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:06,367] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:19:06,367] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:12,314] INFO : Step 13000/500000 rank0 | Train Loss: 12.5156 | Train Loss Unweighted: 4.4627 | Train Loss Components: Classification: 0.0375 | Classification_binary: 3.7303 | Regression_cos_phi: 0.0081 | Regression_energy: 0.3307 | Regression_eta: 0.0250 | Regression_pt: 0.3218 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4123 | Classification_binary: 3.7303 | Regression_cos_phi: 0.0890 | Regression_energy: 3.5585 | Regression_eta: 0.1981 | Regression_pt: 4.4275 | Regression_sin_phi: 0.0999 | LR: 2.88e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0260 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0781 | max_tensorboard_logging: 0.2205 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2706.8 | uss_MB: 1464.6 | child_rss_MB: 22768.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:19:47,338] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:19:47,338] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:25,476] INFO : Step 14000/500000 rank0 | Train Loss: 9.9797 | Train Loss Unweighted: 4.3846 | Train Loss Components: Classification: 0.0338 | Classification_binary: 3.8791 | Regression_cos_phi: 0.0136 | Regression_energy: 0.2169 | Regression_eta: 0.0081 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3715 | Classification_binary: 3.8791 | Regression_cos_phi: 0.1486 | Regression_energy: 2.3344 | Regression_eta: 0.0641 | Regression_pt: 3.0967 | Regression_sin_phi: 0.0852 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7151s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2172 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2707.1 | uss_MB: 1464.8 | child_rss_MB: 22769.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:20:25,478] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:20:25,525] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:20:29,206] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 05:20:29,206] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:10,267] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 05:21:10,267] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:39,564] INFO : Step 15000/500000 rank0 | Train Loss: 10.0383 | Train Loss Unweighted: 4.5144 | Train Loss Components: Classification: 0.0351 | Classification_binary: 4.0122 | Regression_cos_phi: 0.0103 | Regression_energy: 0.2171 | Regression_eta: 0.0118 | Regression_pt: 0.2156 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3861 | Classification_binary: 4.0122 | Regression_cos_phi: 0.1125 | Regression_energy: 2.3365 | Regression_eta: 0.0936 | Regression_pt: 2.9662 | Regression_sin_phi: 0.1312 | LR: 3.17e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7239s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.2194 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.3 | uss_MB: 1468.1 | child_rss_MB: 22766.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:21:52,171] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 05:21:52,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:32,967] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 05:22:32,967] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:52,786] INFO : Step 16000/500000 rank0 | Train Loss: 9.1830 | Train Loss Unweighted: 4.3111 | Train Loss Components: Classification: 0.0416 | Classification_binary: 3.8618 | Regression_cos_phi: 0.0205 | Regression_energy: 0.1841 | Regression_eta: 0.0182 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4566 | Classification_binary: 3.8618 | Regression_cos_phi: 0.2246 | Regression_energy: 1.9815 | Regression_eta: 0.1444 | Regression_pt: 2.4102 | Regression_sin_phi: 0.1039 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0255 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.2198 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.6 | uss_MB: 1468.3 | child_rss_MB: 22768.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 26.3 +rank0 [2026-11-14 05:22:52,787] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:22:52,835] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:23:13,726] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 05:23:13,726] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:23:54,082] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 05:23:54,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:04,954] INFO : Step 17000/500000 rank0 | Train Loss: 9.4302 | Train Loss Unweighted: 4.1930 | Train Loss Components: Classification: 0.0431 | Classification_binary: 3.7172 | Regression_cos_phi: 0.0131 | Regression_energy: 0.1906 | Regression_eta: 0.0101 | Regression_pt: 0.2037 | Regression_sin_phi: 0.0151 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4731 | Classification_binary: 3.7172 | Regression_cos_phi: 0.1435 | Regression_energy: 2.0511 | Regression_eta: 0.0798 | Regression_pt: 2.8029 | Regression_sin_phi: 0.1627 | LR: 3.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7224s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0785 | max_tensorboard_logging: 0.2151 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2706.1 | uss_MB: 1463.9 | child_rss_MB: 22768.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:24:36,096] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 05:24:36,096] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:17,189] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 05:25:17,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:19,027] INFO : Step 18000/500000 rank0 | Train Loss: 9.0094 | Train Loss Unweighted: 4.3172 | Train Loss Components: Classification: 0.0337 | Classification_binary: 3.8911 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1811 | Regression_eta: 0.0102 | Regression_pt: 0.1846 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3697 | Classification_binary: 3.8911 | Regression_cos_phi: 0.0875 | Regression_energy: 1.9493 | Regression_eta: 0.0808 | Regression_pt: 2.5393 | Regression_sin_phi: 0.0918 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7192s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0815 | max_tensorboard_logging: 0.2149 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2707.1 | uss_MB: 1464.8 | child_rss_MB: 22769.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:25:19,029] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:25:19,077] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:25:58,889] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 05:25:58,889] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:32,660] INFO : Step 19000/500000 rank0 | Train Loss: 9.8895 | Train Loss Unweighted: 4.3911 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.8916 | Regression_cos_phi: 0.0101 | Regression_energy: 0.2145 | Regression_eta: 0.0137 | Regression_pt: 0.2178 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3767 | Classification_binary: 3.8916 | Regression_cos_phi: 0.1101 | Regression_energy: 2.3081 | Regression_eta: 0.1084 | Regression_pt: 2.9964 | Regression_sin_phi: 0.0983 | LR: 3.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7240s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0261 | max_loss: 0.0021 | max_backward: 0.0268 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.2182 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.3 | uss_MB: 1468.1 | child_rss_MB: 22770.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 05:26:40,543] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 05:26:40,543] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:21,395] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 05:27:21,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:46,416] INFO : Step 20000/500000 rank0 | Train Loss: 9.1117 | Train Loss Unweighted: 4.1864 | Train Loss Components: Classification: 0.0362 | Classification_binary: 3.7382 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1922 | Regression_eta: 0.0091 | Regression_pt: 0.1889 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3974 | Classification_binary: 3.7382 | Regression_cos_phi: 0.1263 | Regression_energy: 2.0681 | Regression_eta: 0.0725 | Regression_pt: 2.5984 | Regression_sin_phi: 0.1108 | LR: 4.07e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7276s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2204 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2710.6 | uss_MB: 1468.4 | child_rss_MB: 22767.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:27:46,418] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:27:46,465] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:27:46,466] INFO : Running validation on rank0 +rank0 [2026-11-14 05:27:46,466] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:27:46,466] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:27:46,571] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 05:27:46,654] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:27:46,733] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:27:46,781] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:27:46,826] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:27:46,870] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:27:46,916] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:27:46,962] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:27:47,037] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:27:47,115] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:27:47,191] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:27:47,238] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:27:47,284] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:27:47,328] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:27:47,373] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:27:47,419] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:27:47,486] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:27:47,491] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:27:47,492] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:27:47,495] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:27:47,495] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:27:47,497] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:27:47,499] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:27:47,501] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:27:47,502] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:27:47,503] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:27:47,505] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:27:47,506] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:27:47,508] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:27:47,509] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:27:47,511] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:27:47,512] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:27:48,516] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.1117 | Valid Loss=9.8797 | Train Loss Unweighted=4.1864 | Valid Loss Unweighted=4.3140 | Stale=0 | ETA=591.4m +rank0 [2026-11-14 05:27:48,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:27:48,614] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:27:48,623] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:27:48,686] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:27:49,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:27:49,359] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:27:49,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:27:49,693] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:27:49,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:27:50,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:27:50,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:27:50,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:27:50,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:27:50,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:27:50,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:27:50,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:27:51,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:27:51,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:27:51,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:27:51,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:27:51,711] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:27:51,718] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:27:52,092] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:27:52,333] INFO : Plotted number of elements +rank0 [2026-11-14 05:27:53,246] INFO : Plotted jets +rank0 [2026-11-14 05:27:53,942] INFO : Plotted jet ratio +rank0 [2026-11-14 05:28:11,230] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 05:28:11,230] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:52,118] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 05:28:52,118] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:29:07,774] INFO : Step 21000/500000 rank0 | Train Loss: 8.5265 | Train Loss Unweighted: 4.1585 | Train Loss Components: Classification: 0.0376 | Classification_binary: 3.7615 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1655 | Regression_eta: 0.0072 | Regression_pt: 0.1681 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4128 | Classification_binary: 3.7615 | Regression_cos_phi: 0.0911 | Regression_energy: 1.7812 | Regression_eta: 0.0573 | Regression_pt: 2.3121 | Regression_sin_phi: 0.1105 | LR: 4.28e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7357s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0005 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.2220 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2738.5 | uss_MB: 1496.4 | child_rss_MB: 22773.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:29:33,686] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 05:29:33,687] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:14,688] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 05:30:14,689] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:21,344] INFO : Step 22000/500000 rank0 | Train Loss: 7.8826 | Train Loss Unweighted: 3.9374 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.5808 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1513 | Regression_eta: 0.0053 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3243 | Classification_binary: 3.5808 | Regression_cos_phi: 0.0755 | Regression_energy: 1.6284 | Regression_eta: 0.0417 | Regression_pt: 2.1684 | Regression_sin_phi: 0.0635 | LR: 4.50e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7405s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0032 | max_backward: 0.0267 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2223 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2739.6 | uss_MB: 1497.4 | child_rss_MB: 22773.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:30:21,345] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:30:21,394] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:30:56,371] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 05:30:56,371] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:31:35,451] INFO : Step 23000/500000 rank0 | Train Loss: 9.1060 | Train Loss Unweighted: 4.0337 | Train Loss Components: Classification: 0.0303 | Classification_binary: 3.5758 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1932 | Regression_eta: 0.0078 | Regression_pt: 0.2057 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3333 | Classification_binary: 3.5758 | Regression_cos_phi: 0.1155 | Regression_energy: 2.0794 | Regression_eta: 0.0621 | Regression_pt: 2.8302 | Regression_sin_phi: 0.1096 | LR: 4.73e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7296s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0320 | max_optimizer_step: 0.0803 | max_tensorboard_logging: 0.2196 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2745.1 | uss_MB: 1502.9 | child_rss_MB: 22770.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:31:38,514] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 05:31:38,514] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:19,680] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 05:32:19,680] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:49,064] INFO : Step 24000/500000 rank0 | Train Loss: 8.4620 | Train Loss Unweighted: 4.0145 | Train Loss Components: Classification: 0.0306 | Classification_binary: 3.6104 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1708 | Regression_eta: 0.0094 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3357 | Classification_binary: 3.6104 | Regression_cos_phi: 0.1202 | Regression_energy: 1.8378 | Regression_eta: 0.0747 | Regression_pt: 2.3953 | Regression_sin_phi: 0.0878 | LR: 4.97e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7191s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0791 | max_tensorboard_logging: 0.2146 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2745.3 | uss_MB: 1503.1 | child_rss_MB: 22773.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:32:49,066] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:32:49,112] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:33:01,218] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 05:33:01,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:33:42,350] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 05:33:42,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:34:02,862] INFO : Step 25000/500000 rank0 | Train Loss: 9.0820 | Train Loss Unweighted: 4.1736 | Train Loss Components: Classification: 0.0388 | Classification_binary: 3.7257 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1845 | Regression_eta: 0.0146 | Regression_pt: 0.1895 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4264 | Classification_binary: 3.7257 | Regression_cos_phi: 0.1119 | Regression_energy: 1.9855 | Regression_eta: 0.1158 | Regression_pt: 2.6075 | Regression_sin_phi: 0.1093 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7305s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0817 | max_tensorboard_logging: 0.2164 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2738.9 | uss_MB: 1496.7 | child_rss_MB: 22771.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:34:24,211] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 05:34:24,211] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:04,932] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 05:35:04,932] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:16,271] INFO : Step 26000/500000 rank0 | Train Loss: 8.2615 | Train Loss Unweighted: 3.8360 | Train Loss Components: Classification: 0.0370 | Classification_binary: 3.4348 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1677 | Regression_eta: 0.0068 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4064 | Classification_binary: 3.4348 | Regression_cos_phi: 0.0798 | Regression_energy: 1.8050 | Regression_eta: 0.0542 | Regression_pt: 2.3843 | Regression_sin_phi: 0.0970 | LR: 5.47e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7458s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0253 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.2118 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2739.6 | uss_MB: 1497.5 | child_rss_MB: 22773.4 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:35:16,273] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:35:16,321] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:35:45,711] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 05:35:45,711] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:26,541] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 05:36:26,541] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:28,944] INFO : Step 27000/500000 rank0 | Train Loss: 8.2513 | Train Loss Unweighted: 3.8223 | Train Loss Components: Classification: 0.0371 | Classification_binary: 3.4204 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1645 | Regression_eta: 0.0109 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4081 | Classification_binary: 3.4204 | Regression_cos_phi: 0.0726 | Regression_energy: 1.7699 | Regression_eta: 0.0865 | Regression_pt: 2.4238 | Regression_sin_phi: 0.0701 | LR: 5.74e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7154s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0271 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.2080 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.2 | uss_MB: 1501.1 | child_rss_MB: 22772.6 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:37:08,739] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 05:37:08,739] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:43,253] INFO : Step 28000/500000 rank0 | Train Loss: 8.2893 | Train Loss Unweighted: 3.9938 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.6047 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1644 | Regression_eta: 0.0078 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3465 | Classification_binary: 3.6047 | Regression_cos_phi: 0.0979 | Regression_energy: 1.7695 | Regression_eta: 0.0616 | Regression_pt: 2.3445 | Regression_sin_phi: 0.0646 | LR: 6.01e-05 | DataLoad Time: 0.0054s | Model Forward Time: 0.7079s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0800 | max_tensorboard_logging: 0.2077 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.8 | uss_MB: 1501.6 | child_rss_MB: 22772.9 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:37:43,255] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:37:43,301] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:37:50,626] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 05:37:50,626] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:38:31,736] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 05:38:31,736] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:38:57,121] INFO : Step 29000/500000 rank0 | Train Loss: 9.3294 | Train Loss Unweighted: 4.2131 | Train Loss Components: Classification: 0.0416 | Classification_binary: 3.7484 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1960 | Regression_eta: 0.0092 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.4568 | Classification_binary: 3.7484 | Regression_cos_phi: 0.0897 | Regression_energy: 2.1095 | Regression_eta: 0.0733 | Regression_pt: 2.7343 | Regression_sin_phi: 0.1174 | LR: 6.29e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7115s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0278 | max_optimizer_step: 0.0793 | max_tensorboard_logging: 0.2095 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2744.1 | uss_MB: 1501.9 | child_rss_MB: 22773.2 | allocated_MB: 126.1 | reserved_MB: 656.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:39:13,451] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 05:39:13,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:54,550] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 05:39:54,550] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:40:11,233] INFO : Step 30000/500000 rank0 | Train Loss: 8.0436 | Train Loss Unweighted: 3.9024 | Train Loss Components: Classification: 0.0356 | Classification_binary: 3.5233 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1568 | Regression_eta: 0.0128 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3911 | Classification_binary: 3.5233 | Regression_cos_phi: 0.1031 | Regression_energy: 1.6877 | Regression_eta: 0.1015 | Regression_pt: 2.1471 | Regression_sin_phi: 0.0898 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7631s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.1217 | max_tensorboard_logging: 0.2074 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2740.0 | uss_MB: 1497.8 | child_rss_MB: 22772.0 | allocated_MB: 126.1 | reserved_MB: 656.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:40:11,234] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:40:11,282] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:40:11,283] INFO : Running validation on rank0 +rank0 [2026-11-14 05:40:11,283] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:40:11,283] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:40:11,390] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 05:40:11,472] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:40:11,550] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:40:11,596] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:40:11,642] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:40:11,687] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:40:11,733] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:40:11,778] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:40:11,855] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:40:11,932] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:40:12,009] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:40:12,055] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:40:12,100] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:40:12,146] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:40:12,191] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:40:12,237] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:40:12,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:40:12,309] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:40:12,311] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:40:12,314] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:40:12,314] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:40:12,316] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:40:12,319] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:40:12,320] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:40:12,321] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:40:12,324] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:40:12,325] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:40:12,327] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:40:12,327] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:40:12,330] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:40:12,331] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:40:12,332] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:40:13,313] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.0436 | Valid Loss=9.5466 | Train Loss Unweighted=3.9024 | Valid Loss Unweighted=4.1642 | Stale=0 | ETA=580.5m +rank0 [2026-11-14 05:40:13,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:40:13,409] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:40:13,428] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:40:13,487] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:40:14,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:40:14,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:40:14,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:40:14,500] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:40:14,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:40:14,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:40:14,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:40:15,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:40:15,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:40:15,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:40:15,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:40:15,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:40:15,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:40:16,116] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:40:16,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:40:16,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:40:16,529] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:40:16,543] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:40:16,930] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:40:17,170] INFO : Plotted number of elements +rank0 [2026-11-14 05:40:18,098] INFO : Plotted jets +rank0 [2026-11-14 05:40:18,792] INFO : Plotted jet ratio +rank0 [2026-11-14 05:40:44,855] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 05:40:44,855] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:26,108] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 05:41:26,108] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:33,371] INFO : Step 31000/500000 rank0 | Train Loss: 7.6375 | Train Loss Unweighted: 3.7111 | Train Loss Components: Classification: 0.0303 | Classification_binary: 3.3545 | Regression_cos_phi: 0.0050 | Regression_energy: 0.1497 | Regression_eta: 0.0073 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3325 | Classification_binary: 3.3545 | Regression_cos_phi: 0.0546 | Regression_energy: 1.6108 | Regression_eta: 0.0576 | Regression_pt: 2.1030 | Regression_sin_phi: 0.1246 | LR: 6.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7437s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0334 | max_optimizer_step: 0.0828 | max_tensorboard_logging: 0.2184 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2864.8 | uss_MB: 1622.6 | child_rss_MB: 22770.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:42:08,014] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 05:42:08,014] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:42:47,454] INFO : Step 32000/500000 rank0 | Train Loss: 8.9294 | Train Loss Unweighted: 4.0396 | Train Loss Components: Classification: 0.0341 | Classification_binary: 3.5952 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1938 | Regression_eta: 0.0106 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3745 | Classification_binary: 3.5952 | Regression_cos_phi: 0.0607 | Regression_energy: 2.0858 | Regression_eta: 0.0842 | Regression_pt: 2.6374 | Regression_sin_phi: 0.0916 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7456s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0022 | max_backward: 0.0332 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.2161 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2864.8 | uss_MB: 1622.6 | child_rss_MB: 22770.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:42:47,456] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:42:47,504] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:42:50,015] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 05:42:50,015] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:43:31,534] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 05:43:31,534] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:01,612] INFO : Step 33000/500000 rank0 | Train Loss: 8.5106 | Train Loss Unweighted: 4.1768 | Train Loss Components: Classification: 0.0351 | Classification_binary: 3.7816 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1631 | Regression_eta: 0.0109 | Regression_pt: 0.1660 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3860 | Classification_binary: 3.7816 | Regression_cos_phi: 0.1066 | Regression_energy: 1.7546 | Regression_eta: 0.0865 | Regression_pt: 2.2842 | Regression_sin_phi: 0.1111 | LR: 7.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7309s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0795 | max_tensorboard_logging: 0.2218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.5 | uss_MB: 1522.3 | child_rss_MB: 22771.0 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 05:44:13,142] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:44:13,142] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:54,071] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:44:54,071] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:15,191] INFO : Step 34000/500000 rank0 | Train Loss: 9.7737 | Train Loss Unweighted: 4.2839 | Train Loss Components: Classification: 0.0352 | Classification_binary: 3.7838 | Regression_cos_phi: 0.0104 | Regression_energy: 0.2133 | Regression_eta: 0.0147 | Regression_pt: 0.2135 | Regression_sin_phi: 0.0130 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3867 | Classification_binary: 3.7838 | Regression_cos_phi: 0.1135 | Regression_energy: 2.2959 | Regression_eta: 0.1169 | Regression_pt: 2.9372 | Regression_sin_phi: 0.1397 | LR: 7.83e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7158s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0269 | max_optimizer_step: 0.0793 | max_tensorboard_logging: 0.2104 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2765.0 | uss_MB: 1522.8 | child_rss_MB: 22770.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:45:15,193] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:45:15,241] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:45:35,792] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:45:35,793] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:16,984] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:46:16,985] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:29,097] INFO : Step 35000/500000 rank0 | Train Loss: 8.2933 | Train Loss Unweighted: 4.0214 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.6335 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1719 | Regression_eta: 0.0081 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3107 | Classification_binary: 3.6335 | Regression_cos_phi: 0.0708 | Regression_energy: 1.8498 | Regression_eta: 0.0645 | Regression_pt: 2.3042 | Regression_sin_phi: 0.0597 | LR: 8.16e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7135s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0281 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2094 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2769.8 | uss_MB: 1527.6 | child_rss_MB: 22771.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:46:58,241] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:46:58,241] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:47:39,152] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:47:39,152] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:47:42,136] INFO : Step 36000/500000 rank0 | Train Loss: 7.9417 | Train Loss Unweighted: 3.6807 | Train Loss Components: Classification: 0.0360 | Classification_binary: 3.2924 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1595 | Regression_eta: 0.0091 | Regression_pt: 0.1625 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3957 | Classification_binary: 3.2924 | Regression_cos_phi: 0.1200 | Regression_energy: 1.7165 | Regression_eta: 0.0723 | Regression_pt: 2.2354 | Regression_sin_phi: 0.1093 | LR: 8.50e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7163s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2102 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2769.8 | uss_MB: 1527.7 | child_rss_MB: 22771.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:47:42,137] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:47:42,184] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:48:20,912] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:48:20,912] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:55,195] INFO : Step 37000/500000 rank0 | Train Loss: 8.1863 | Train Loss Unweighted: 3.7778 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.3774 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1694 | Regression_eta: 0.0081 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3119 | Classification_binary: 3.3774 | Regression_cos_phi: 0.1156 | Regression_energy: 1.8233 | Regression_eta: 0.0645 | Regression_pt: 2.3660 | Regression_sin_phi: 0.1276 | LR: 8.85e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7507s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2426 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2769.9 | uss_MB: 1527.8 | child_rss_MB: 22772.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:49:01,681] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:49:01,681] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:49:41,833] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:49:41,833] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:06,986] INFO : Step 38000/500000 rank0 | Train Loss: 9.1903 | Train Loss Unweighted: 4.1422 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.6797 | Regression_cos_phi: 0.0196 | Regression_energy: 0.1908 | Regression_eta: 0.0154 | Regression_pt: 0.1890 | Regression_sin_phi: 0.0137 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3729 | Classification_binary: 3.6797 | Regression_cos_phi: 0.2140 | Regression_energy: 2.0536 | Regression_eta: 0.1223 | Regression_pt: 2.6004 | Regression_sin_phi: 0.1474 | LR: 9.21e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7058s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0022 | max_backward: 0.0256 | max_optimizer_step: 0.0787 | max_tensorboard_logging: 0.2086 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2765.5 | uss_MB: 1523.4 | child_rss_MB: 22773.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:50:06,987] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:50:07,034] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:50:22,296] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:50:22,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:03,256] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:51:03,257] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:20,260] INFO : Step 39000/500000 rank0 | Train Loss: 8.2477 | Train Loss Unweighted: 3.7351 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.3249 | Regression_cos_phi: 0.0132 | Regression_energy: 0.1714 | Regression_eta: 0.0092 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3052 | Classification_binary: 3.3249 | Regression_cos_phi: 0.1443 | Regression_energy: 1.8441 | Regression_eta: 0.0730 | Regression_pt: 2.4170 | Regression_sin_phi: 0.1391 | LR: 9.57e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7098s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2083 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2770.4 | uss_MB: 1528.2 | child_rss_MB: 22772.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:51:45,182] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:51:45,182] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:26,070] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:52:26,070] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:34,191] INFO : Step 40000/500000 rank0 | Train Loss: 8.4592 | Train Loss Unweighted: 3.7702 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.3415 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1854 | Regression_eta: 0.0127 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3771 | Classification_binary: 3.3415 | Regression_cos_phi: 0.1011 | Regression_energy: 1.9954 | Regression_eta: 0.1008 | Regression_pt: 2.4395 | Regression_sin_phi: 0.1038 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0022 | max_backward: 0.0257 | max_optimizer_step: 0.0804 | max_tensorboard_logging: 0.2074 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2770.7 | uss_MB: 1528.6 | child_rss_MB: 22772.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:52:34,192] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:52:34,239] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:52:34,240] INFO : Running validation on rank0 +rank0 [2026-11-14 05:52:34,240] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:52:34,240] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:52:34,348] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:52:34,429] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:52:34,507] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:52:34,555] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:52:34,601] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:52:34,647] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:52:34,693] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:52:34,739] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:52:34,804] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:52:34,881] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:52:34,957] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:52:35,004] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:52:35,050] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:52:35,095] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:52:35,141] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:52:35,186] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:52:35,252] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:52:35,257] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:52:35,258] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:52:35,260] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:52:35,262] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:52:35,263] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:52:35,265] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:52:35,266] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:52:35,268] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:52:35,269] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:52:35,271] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:52:35,272] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:52:35,274] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:52:35,275] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:52:35,277] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:52:35,278] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:52:36,306] INFO : VALIDATION | Step=40000/500000 | Train Loss=8.4592 | Valid Loss=9.1452 | Train Loss Unweighted=3.7702 | Valid Loss Unweighted=3.9775 | Stale=0 | ETA=568.5m +rank0 [2026-11-14 05:52:36,306] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:52:36,398] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:52:36,406] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:36,453] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:52:36,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:52:37,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:52:37,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:52:37,363] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:52:37,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:52:37,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:52:37,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:52:38,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:52:38,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:52:38,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:52:38,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:52:38,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:52:38,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:52:38,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:52:39,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:52:39,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:52:39,353] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-14 05:52:39,359] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:39,747] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:39,997] INFO : Plotted number of elements +rank0 [2026-11-14 05:52:40,889] INFO : Plotted jets +rank0 [2026-11-14 05:52:41,585] INFO : Plotted jet ratio +rank0 [2026-11-14 05:53:15,196] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:53:15,196] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:53:54,562] INFO : Step 41000/500000 rank0 | Train Loss: 7.7864 | Train Loss Unweighted: 3.7823 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.4175 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1520 | Regression_eta: 0.0095 | Regression_pt: 0.1540 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3207 | Classification_binary: 3.4175 | Regression_cos_phi: 0.1139 | Regression_energy: 1.6356 | Regression_eta: 0.0751 | Regression_pt: 2.1189 | Regression_sin_phi: 0.1047 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0798 | max_tensorboard_logging: 0.2116 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2789.5 | uss_MB: 1547.4 | child_rss_MB: 22773.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:53:56,466] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:53:56,466] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:37,472] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:54:37,472] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:55:08,212] INFO : Step 42000/500000 rank0 | Train Loss: 9.1765 | Train Loss Unweighted: 4.1269 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.6690 | Regression_cos_phi: 0.0122 | Regression_energy: 0.1873 | Regression_eta: 0.0123 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0156 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3093 | Classification_binary: 3.6690 | Regression_cos_phi: 0.1334 | Regression_energy: 2.0152 | Regression_eta: 0.0973 | Regression_pt: 2.7838 | Regression_sin_phi: 0.1684 | LR: 1.07e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7094s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0788 | max_tensorboard_logging: 0.2063 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2790.3 | uss_MB: 1548.2 | child_rss_MB: 22774.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:55:08,214] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:55:08,262] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:55:19,041] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:55:19,042] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:00,370] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:56:00,371] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:21,677] INFO : Step 43000/500000 rank0 | Train Loss: 7.5563 | Train Loss Unweighted: 3.4113 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.0382 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1605 | Regression_eta: 0.0063 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2649 | Classification_binary: 3.0382 | Regression_cos_phi: 0.0516 | Regression_energy: 1.7274 | Regression_eta: 0.0497 | Regression_pt: 2.3640 | Regression_sin_phi: 0.0605 | LR: 1.11e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7056s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0802 | max_tensorboard_logging: 0.2020 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.5 | uss_MB: 1552.4 | child_rss_MB: 22774.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:56:41,522] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:56:41,522] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:22,717] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:57:22,717] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:35,463] INFO : Step 44000/500000 rank0 | Train Loss: 7.1248 | Train Loss Unweighted: 3.4840 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.1502 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1382 | Regression_eta: 0.0128 | Regression_pt: 0.1372 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3449 | Classification_binary: 3.1502 | Regression_cos_phi: 0.0645 | Regression_energy: 1.4873 | Regression_eta: 0.1016 | Regression_pt: 1.8870 | Regression_sin_phi: 0.0891 | LR: 1.15e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7186s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.2137 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.9 | uss_MB: 1552.8 | child_rss_MB: 22773.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:57:35,465] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:57:35,512] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:58:04,395] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:58:04,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:45,792] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 05:58:45,793] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:49,744] INFO : Step 45000/500000 rank0 | Train Loss: 7.9170 | Train Loss Unweighted: 3.7282 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.3424 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1652 | Regression_eta: 0.0150 | Regression_pt: 0.1525 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3461 | Classification_binary: 3.3424 | Regression_cos_phi: 0.1193 | Regression_energy: 1.7777 | Regression_eta: 0.1189 | Regression_pt: 2.0980 | Regression_sin_phi: 0.1145 | LR: 1.19e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7042s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2001 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.9 | uss_MB: 1552.8 | child_rss_MB: 22775.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:59:28,203] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 05:59:28,203] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:04,016] INFO : Step 46000/500000 rank0 | Train Loss: 7.5449 | Train Loss Unweighted: 3.8439 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.5056 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1384 | Regression_eta: 0.0110 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3143 | Classification_binary: 3.5056 | Regression_cos_phi: 0.0974 | Regression_energy: 1.4897 | Regression_eta: 0.0872 | Regression_pt: 1.9356 | Regression_sin_phi: 0.1151 | LR: 1.23e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7117s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0820 | max_tensorboard_logging: 0.2001 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2790.4 | uss_MB: 1548.3 | child_rss_MB: 22773.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:00:04,018] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 06:00:04,066] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:00:10,292] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 06:00:10,292] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:51,772] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 06:00:51,772] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:18,399] INFO : Step 47000/500000 rank0 | Train Loss: 7.2733 | Train Loss Unweighted: 3.5060 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.1627 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1432 | Regression_eta: 0.0103 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3204 | Classification_binary: 3.1627 | Regression_cos_phi: 0.0677 | Regression_energy: 1.5404 | Regression_eta: 0.0816 | Regression_pt: 2.0106 | Regression_sin_phi: 0.0898 | LR: 1.27e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7085s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0806 | max_tensorboard_logging: 0.2059 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.1 | uss_MB: 1552.0 | child_rss_MB: 22774.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:01:33,421] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 06:01:33,421] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:14,157] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 06:02:14,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:31,739] INFO : Step 48000/500000 rank0 | Train Loss: 7.1753 | Train Loss Unweighted: 3.5857 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.2580 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1295 | Regression_eta: 0.0118 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2908 | Classification_binary: 3.2580 | Regression_cos_phi: 0.1085 | Regression_energy: 1.3930 | Regression_eta: 0.0937 | Regression_pt: 1.9152 | Regression_sin_phi: 0.1161 | LR: 1.31e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7172s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0306 | max_loss: 0.0021 | max_backward: 0.0271 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2061 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2794.2 | uss_MB: 1552.1 | child_rss_MB: 22774.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:02:31,741] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:02:31,787] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:02:56,240] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 06:02:56,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:36,654] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 06:03:36,654] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:44,938] INFO : Step 49000/500000 rank0 | Train Loss: 7.3959 | Train Loss Unweighted: 3.8290 | Train Loss Components: Classification: 0.0245 | Classification_binary: 3.5047 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1386 | Regression_eta: 0.0067 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2697 | Classification_binary: 3.5047 | Regression_cos_phi: 0.0880 | Regression_energy: 1.4912 | Regression_eta: 0.0534 | Regression_pt: 1.9000 | Regression_sin_phi: 0.0888 | LR: 1.36e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7146s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.2144 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2794.2 | uss_MB: 1552.1 | child_rss_MB: 22775.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:04:18,092] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 06:04:18,092] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:59,090] INFO : Step 50000/500000 rank0 | Train Loss: 7.8626 | Train Loss Unweighted: 3.5490 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.1553 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1626 | Regression_eta: 0.0160 | Regression_pt: 0.1696 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3308 | Classification_binary: 3.1553 | Regression_cos_phi: 0.0825 | Regression_energy: 1.7502 | Regression_eta: 0.1270 | Regression_pt: 2.3335 | Regression_sin_phi: 0.0833 | LR: 1.40e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0234 | max_loss: 0.0021 | max_backward: 0.0264 | max_optimizer_step: 0.0786 | max_tensorboard_logging: 0.2141 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2789.7 | uss_MB: 1547.6 | child_rss_MB: 22773.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:04:59,092] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:04:59,139] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:04:59,139] INFO : Running validation on rank0 +rank0 [2026-11-14 06:04:59,140] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:04:59,140] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:04:59,246] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 06:04:59,326] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:04:59,404] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:04:59,454] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:04:59,499] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:04:59,544] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:04:59,590] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:04:59,636] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:04:59,702] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:04:59,779] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:04:59,857] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:04:59,904] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:04:59,950] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:04:59,995] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:05:00,043] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:05:00,089] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:05:00,155] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:05:00,159] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:05:00,161] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:05:00,162] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:05:00,164] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:05:00,165] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:05:00,167] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:05:00,169] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:05:00,170] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:05:00,172] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:05:00,173] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:05:00,175] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:05:00,177] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:05:00,178] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:05:00,180] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:05:00,181] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:05:01,160] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.8626 | Valid Loss=9.0423 | Train Loss Unweighted=3.5490 | Valid Loss Unweighted=3.8565 | Stale=0 | ETA=556.7m +rank0 [2026-11-14 06:05:01,160] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:05:01,264] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:05:01,272] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:05:01,339] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:05:01,861] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:05:02,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:05:02,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:05:02,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:05:02,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:05:02,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:05:02,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:05:02,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:05:03,142] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:05:03,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:05:03,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:05:03,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:05:03,806] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:05:03,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:05:04,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:05:04,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:05:04,393] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:05:04,399] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:05:04,771] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:05:05,015] INFO : Plotted number of elements +rank0 [2026-11-14 06:05:05,937] INFO : Plotted jets +rank0 [2026-11-14 06:05:06,632] INFO : Plotted jet ratio +rank0 [2026-11-14 06:05:08,400] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 06:05:08,400] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:49,887] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 06:05:49,887] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:21,388] INFO : Step 51000/500000 rank0 | Train Loss: 7.0579 | Train Loss Unweighted: 3.4916 | Train Loss Components: Classification: 0.0281 | Classification_binary: 3.1659 | Regression_cos_phi: 0.0136 | Regression_energy: 0.1340 | Regression_eta: 0.0078 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3090 | Classification_binary: 3.1659 | Regression_cos_phi: 0.1487 | Regression_energy: 1.4415 | Regression_eta: 0.0619 | Regression_pt: 1.8401 | Regression_sin_phi: 0.0909 | LR: 1.44e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7457s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0300 | max_loss: 0.0021 | max_backward: 0.0270 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.2096 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2923.6 | uss_MB: 1681.5 | child_rss_MB: 22773.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:31,894] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 06:06:31,894] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:13,297] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 06:07:13,297] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:35,677] INFO : Step 52000/500000 rank0 | Train Loss: 7.3492 | Train Loss Unweighted: 3.5153 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.1663 | Regression_cos_phi: 0.0157 | Regression_energy: 0.1492 | Regression_eta: 0.0073 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2721 | Classification_binary: 3.1663 | Regression_cos_phi: 0.1720 | Regression_energy: 1.6054 | Regression_eta: 0.0580 | Regression_pt: 2.0206 | Regression_sin_phi: 0.0547 | LR: 1.49e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7358s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0813 | max_tensorboard_logging: 0.2201 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2923.6 | uss_MB: 1681.5 | child_rss_MB: 22774.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:07:35,678] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 06:07:35,727] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 06:07:55,350] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 06:07:55,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:37,000] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 06:08:37,000] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:50,327] INFO : Step 53000/500000 rank0 | Train Loss: 6.8211 | Train Loss Unweighted: 3.4596 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.1521 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1242 | Regression_eta: 0.0087 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3295 | Classification_binary: 3.1521 | Regression_cos_phi: 0.1147 | Regression_energy: 1.3362 | Regression_eta: 0.0688 | Regression_pt: 1.7277 | Regression_sin_phi: 0.0921 | LR: 1.53e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7357s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0799 | max_tensorboard_logging: 0.2154 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2923.6 | uss_MB: 1681.5 | child_rss_MB: 22771.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:09:19,049] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 06:09:19,049] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:00,285] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 06:10:00,286] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:04,467] INFO : Step 54000/500000 rank0 | Train Loss: 6.3049 | Train Loss Unweighted: 3.2581 | Train Loss Components: Classification: 0.0284 | Classification_binary: 2.9809 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1139 | Regression_eta: 0.0062 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3122 | Classification_binary: 2.9809 | Regression_cos_phi: 0.0659 | Regression_energy: 1.2254 | Regression_eta: 0.0493 | Regression_pt: 1.6128 | Regression_sin_phi: 0.0582 | LR: 1.58e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0820 | max_tensorboard_logging: 0.2161 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2814.9 | uss_MB: 1572.8 | child_rss_MB: 22774.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:10:04,468] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 06:10:04,515] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:10:41,333] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 06:10:41,333] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:17,719] INFO : Step 55000/500000 rank0 | Train Loss: 6.1466 | Train Loss Unweighted: 3.3085 | Train Loss Components: Classification: 0.0220 | Classification_binary: 3.0494 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1077 | Regression_eta: 0.0068 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2415 | Classification_binary: 3.0494 | Regression_cos_phi: 0.0795 | Regression_energy: 1.1592 | Regression_eta: 0.0537 | Regression_pt: 1.4763 | Regression_sin_phi: 0.0870 | LR: 1.62e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7342s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2123 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.0 | uss_MB: 1574.9 | child_rss_MB: 22775.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:24,029] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 06:11:24,053] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:12:06,856] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 06:12:06,856] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:12:34,262] INFO : Step 56000/500000 rank0 | Train Loss: 5.9982 | Train Loss Unweighted: 3.1164 | Train Loss Components: Classification: 0.0302 | Classification_binary: 2.8507 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1056 | Regression_eta: 0.0099 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3319 | Classification_binary: 2.8507 | Regression_cos_phi: 0.0803 | Regression_energy: 1.1369 | Regression_eta: 0.0783 | Regression_pt: 1.4189 | Regression_sin_phi: 0.1012 | LR: 1.67e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7456s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0333 | max_optimizer_step: 0.0806 | max_tensorboard_logging: 0.2305 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.1 | uss_MB: 1574.9 | child_rss_MB: 22775.6 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:12:34,264] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 06:12:34,339] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:12:49,105] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 06:12:49,105] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:30,708] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 06:13:30,708] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:48,983] INFO : Step 57000/500000 rank0 | Train Loss: 5.7304 | Train Loss Unweighted: 3.0672 | Train Loss Components: Classification: 0.0266 | Classification_binary: 2.8237 | Regression_cos_phi: 0.0071 | Regression_energy: 0.0991 | Regression_eta: 0.0051 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2920 | Classification_binary: 2.8237 | Regression_cos_phi: 0.0773 | Regression_energy: 1.0667 | Regression_eta: 0.0403 | Regression_pt: 1.3460 | Regression_sin_phi: 0.0846 | LR: 1.72e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7106s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0270 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2080 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2818.1 | uss_MB: 1576.0 | child_rss_MB: 22774.6 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:13,006] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 06:14:13,007] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:14:54,940] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 06:14:54,940] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:03,995] INFO : Step 58000/500000 rank0 | Train Loss: 6.3357 | Train Loss Unweighted: 3.4307 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.1621 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1058 | Regression_eta: 0.0139 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2656 | Classification_binary: 3.1621 | Regression_cos_phi: 0.0778 | Regression_energy: 1.1381 | Regression_eta: 0.1103 | Regression_pt: 1.4495 | Regression_sin_phi: 0.1323 | LR: 1.76e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7188s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0021 | max_backward: 0.0269 | max_optimizer_step: 0.0826 | max_tensorboard_logging: 0.2125 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2814.2 | uss_MB: 1572.1 | child_rss_MB: 22773.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:15:03,997] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 06:15:04,048] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 06:15:37,160] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 06:15:37,161] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:16:18,603] INFO : Step 59000/500000 rank0 | Train Loss: 6.8693 | Train Loss Unweighted: 3.4078 | Train Loss Components: Classification: 0.0323 | Classification_binary: 3.0910 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1282 | Regression_eta: 0.0098 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3553 | Classification_binary: 3.0910 | Regression_cos_phi: 0.0855 | Regression_energy: 1.3798 | Regression_eta: 0.0773 | Regression_pt: 1.7835 | Regression_sin_phi: 0.0969 | LR: 1.81e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7607s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0816 | max_tensorboard_logging: 0.2108 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.6 | uss_MB: 1575.5 | child_rss_MB: 22772.7 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:16:19,328] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 06:16:19,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:00,823] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 06:17:00,823] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:33,459] INFO : Step 60000/500000 rank0 | Train Loss: 6.1460 | Train Loss Unweighted: 3.2364 | Train Loss Components: Classification: 0.0296 | Classification_binary: 2.9674 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1051 | Regression_eta: 0.0103 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3249 | Classification_binary: 2.9674 | Regression_cos_phi: 0.1245 | Regression_energy: 1.1310 | Regression_eta: 0.0817 | Regression_pt: 1.4003 | Regression_sin_phi: 0.1162 | LR: 1.86e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7277s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0270 | max_loss: 0.0021 | max_backward: 0.0270 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.2150 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.8 | uss_MB: 1575.7 | child_rss_MB: 22771.5 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:17:33,461] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 06:17:33,509] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 06:17:33,511] INFO : Running validation on rank0 +rank0 [2026-11-14 06:17:33,511] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:17:33,511] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:17:33,649] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 06:17:33,732] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:17:33,830] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:17:33,879] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:17:33,924] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:17:33,969] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:17:34,016] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:17:34,062] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:17:34,128] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:17:34,205] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:17:34,282] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:17:34,329] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:17:34,376] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:17:34,423] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:17:34,469] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:17:34,515] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:17:34,583] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:17:34,587] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:17:34,589] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:17:34,591] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:17:34,592] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:17:34,594] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:17:34,596] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:17:34,597] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:17:34,598] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:17:34,600] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:17:34,601] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:17:34,603] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:17:34,604] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:17:34,606] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:17:34,608] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:17:34,609] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:17:35,542] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.1460 | Valid Loss=9.5209 | Train Loss Unweighted=3.2364 | Valid Loss Unweighted=3.9083 | Stale=1 | ETA=545.8m +rank0 [2026-11-14 06:17:35,542] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:17:35,641] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:17:35,650] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:17:35,697] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:17:36,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:17:36,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:17:36,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:17:36,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:17:36,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:17:36,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:17:37,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:17:37,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:17:37,450] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:17:37,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:17:37,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:17:37,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:17:38,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:17:38,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:17:38,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:17:38,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:17:38,691] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:17:38,698] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:17:39,074] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:17:39,317] INFO : Plotted number of elements +rank0 [2026-11-14 06:17:40,221] INFO : Plotted jets +rank0 [2026-11-14 06:17:40,920] INFO : Plotted jet ratio +rank0 [2026-11-14 06:17:51,318] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 06:17:51,318] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:32,944] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 06:18:32,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:56,071] INFO : Step 61000/500000 rank0 | Train Loss: 6.2206 | Train Loss Unweighted: 3.3921 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.1288 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1023 | Regression_eta: 0.0175 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3058 | Classification_binary: 3.1288 | Regression_cos_phi: 0.0864 | Regression_energy: 1.1013 | Regression_eta: 0.1386 | Regression_pt: 1.3818 | Regression_sin_phi: 0.0779 | LR: 1.91e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7363s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0021 | max_backward: 0.0271 | max_optimizer_step: 0.0801 | max_tensorboard_logging: 0.2124 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2944.2 | uss_MB: 1702.1 | child_rss_MB: 22764.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:19:15,118] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 06:19:15,118] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:56,570] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 06:19:56,571] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:10,514] INFO : Step 62000/500000 rank0 | Train Loss: 6.4635 | Train Loss Unweighted: 3.2660 | Train Loss Components: Classification: 0.0279 | Classification_binary: 2.9747 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1219 | Regression_eta: 0.0058 | Regression_pt: 0.1211 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3061 | Classification_binary: 2.9747 | Regression_cos_phi: 0.0933 | Regression_energy: 1.3117 | Regression_eta: 0.0459 | Regression_pt: 1.6663 | Regression_sin_phi: 0.0655 | LR: 1.95e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7370s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0823 | max_tensorboard_logging: 0.2188 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.9 | uss_MB: 1597.8 | child_rss_MB: 22762.8 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:20:10,516] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 06:20:10,569] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 06:20:38,796] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 06:20:38,796] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:20,525] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 06:21:20,525] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:25,262] INFO : Step 63000/500000 rank0 | Train Loss: 5.7807 | Train Loss Unweighted: 3.1847 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.9480 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0944 | Regression_eta: 0.0059 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2716 | Classification_binary: 2.9480 | Regression_cos_phi: 0.0539 | Regression_energy: 1.0161 | Regression_eta: 0.0464 | Regression_pt: 1.3621 | Regression_sin_phi: 0.0825 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7241s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0276 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.2072 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2843.5 | uss_MB: 1601.4 | child_rss_MB: 22763.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:22:02,414] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 06:22:02,414] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:39,267] INFO : Step 64000/500000 rank0 | Train Loss: 5.9483 | Train Loss Unweighted: 2.9071 | Train Loss Components: Classification: 0.0280 | Classification_binary: 2.6271 | Regression_cos_phi: 0.0120 | Regression_energy: 0.1098 | Regression_eta: 0.0090 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3079 | Classification_binary: 2.6271 | Regression_cos_phi: 0.1311 | Regression_energy: 1.1812 | Regression_eta: 0.0709 | Regression_pt: 1.4909 | Regression_sin_phi: 0.1392 | LR: 2.05e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7693s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0256 | max_optimizer_step: 0.0805 | max_tensorboard_logging: 0.2067 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2843.6 | uss_MB: 1601.5 | child_rss_MB: 22766.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:22:39,268] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 06:22:39,318] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 06:22:44,177] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 06:22:44,177] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:24,207] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 06:23:24,207] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:52,393] INFO : Step 65000/500000 rank0 | Train Loss: 6.7107 | Train Loss Unweighted: 3.3816 | Train Loss Components: Classification: 0.0334 | Classification_binary: 3.0783 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1214 | Regression_eta: 0.0049 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.3665 | Classification_binary: 3.0783 | Regression_cos_phi: 0.1012 | Regression_energy: 1.3061 | Regression_eta: 0.0390 | Regression_pt: 1.7140 | Regression_sin_phi: 0.1056 | LR: 2.10e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0244 | max_loss: 0.0021 | max_backward: 0.0273 | max_optimizer_step: 0.0807 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2844.5 | uss_MB: 1602.4 | child_rss_MB: 22763.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:24:06,523] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 06:24:06,523] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:47,940] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 06:24:47,941] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:06,708] INFO : Step 66000/500000 rank0 | Train Loss: 6.2999 | Train Loss Unweighted: 3.3176 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.0449 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1097 | Regression_eta: 0.0078 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2983 | Classification_binary: 3.0449 | Regression_cos_phi: 0.0833 | Regression_energy: 1.1803 | Regression_eta: 0.0617 | Regression_pt: 1.5430 | Regression_sin_phi: 0.0882 | LR: 2.15e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7158s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0022 | max_backward: 0.0269 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2080 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.9 | uss_MB: 1597.8 | child_rss_MB: 22764.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:25:06,710] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 06:25:06,761] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 06:25:30,057] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 06:25:30,057] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:11,532] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 06:26:11,533] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:21,198] INFO : Step 67000/500000 rank0 | Train Loss: 6.3735 | Train Loss Unweighted: 3.2323 | Train Loss Components: Classification: 0.0272 | Classification_binary: 2.9449 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1197 | Regression_eta: 0.0076 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2985 | Classification_binary: 2.9449 | Regression_cos_phi: 0.1006 | Regression_energy: 1.2886 | Regression_eta: 0.0605 | Regression_pt: 1.6074 | Regression_sin_phi: 0.0729 | LR: 2.20e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7074s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2053 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2843.5 | uss_MB: 1601.4 | child_rss_MB: 22765.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 30.3 +rank0 [2026-11-14 06:26:53,923] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 06:26:53,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:36,105] INFO : Step 68000/500000 rank0 | Train Loss: 6.3533 | Train Loss Unweighted: 3.3041 | Train Loss Components: Classification: 0.0244 | Classification_binary: 3.0251 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1137 | Regression_eta: 0.0084 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2683 | Classification_binary: 3.0251 | Regression_cos_phi: 0.1022 | Regression_energy: 1.2235 | Regression_eta: 0.0665 | Regression_pt: 1.5711 | Regression_sin_phi: 0.0967 | LR: 2.25e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7179s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0035 | max_backward: 0.0267 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.2084 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2843.6 | uss_MB: 1601.5 | child_rss_MB: 22765.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:27:36,107] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 06:27:36,153] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 06:27:36,299] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 06:27:36,299] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:17,682] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 06:28:17,682] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:50,451] INFO : Step 69000/500000 rank0 | Train Loss: 5.4821 | Train Loss Unweighted: 3.0174 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.7926 | Regression_cos_phi: 0.0106 | Regression_energy: 0.0920 | Regression_eta: 0.0050 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2030 | Classification_binary: 2.7926 | Regression_cos_phi: 0.1163 | Regression_energy: 0.9899 | Regression_eta: 0.0397 | Regression_pt: 1.2759 | Regression_sin_phi: 0.0648 | LR: 2.30e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7072s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0022 | max_backward: 0.0269 | max_optimizer_step: 0.0834 | max_tensorboard_logging: 0.1943 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2843.8 | uss_MB: 1601.7 | child_rss_MB: 22766.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:28:59,674] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 06:28:59,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:40,971] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 06:29:40,971] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:04,817] INFO : Step 70000/500000 rank0 | Train Loss: 5.4883 | Train Loss Unweighted: 3.1045 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.8848 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0907 | Regression_eta: 0.0081 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2230 | Classification_binary: 2.8848 | Regression_cos_phi: 0.0757 | Regression_energy: 0.9764 | Regression_eta: 0.0641 | Regression_pt: 1.1783 | Regression_sin_phi: 0.0861 | LR: 2.35e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0270 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.1949 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2840.0 | uss_MB: 1597.9 | child_rss_MB: 22770.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:30:04,818] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 06:30:04,864] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 06:30:04,865] INFO : Running validation on rank0 +rank0 [2026-11-14 06:30:04,865] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:30:04,865] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:30:04,973] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 06:30:05,054] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:30:05,131] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:30:05,181] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:30:05,227] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:30:05,272] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:30:05,317] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:30:05,364] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:30:05,429] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:30:05,507] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:30:05,584] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:30:05,632] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:30:05,678] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:30:05,723] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:30:05,769] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:30:05,815] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:30:05,879] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:30:05,883] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:30:05,885] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:30:05,886] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:30:05,888] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:30:05,889] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:30:05,892] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:30:05,892] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:30:05,894] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:30:05,896] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:30:05,897] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:30:05,899] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:30:05,900] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:30:05,902] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:30:05,903] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:30:05,905] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:30:06,843] INFO : VALIDATION | Step=70000/500000 | Train Loss=5.4883 | Valid Loss=9.6698 | Train Loss Unweighted=3.1045 | Valid Loss Unweighted=4.0003 | Stale=2 | ETA=534.1m +rank0 [2026-11-14 06:30:06,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:30:06,935] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:30:06,944] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:06,991] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:30:07,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:30:07,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:30:07,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:30:07,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:30:08,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:30:08,238] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:30:08,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:30:08,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:30:08,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:30:08,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:30:09,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:30:09,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:30:09,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:30:09,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:30:09,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:30:09,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:30:09,942] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:30:09,949] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:10,334] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:10,577] INFO : Plotted number of elements +rank0 [2026-11-14 06:30:11,502] INFO : Plotted jets +rank0 [2026-11-14 06:30:12,196] INFO : Plotted jet ratio +rank0 [2026-11-14 06:30:30,916] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 06:30:30,917] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:11,907] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 06:31:11,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:26,124] INFO : Step 71000/500000 rank0 | Train Loss: 4.7648 | Train Loss Unweighted: 2.7346 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.5480 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0744 | Regression_eta: 0.0064 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2358 | Classification_binary: 2.5480 | Regression_cos_phi: 0.0581 | Regression_energy: 0.8008 | Regression_eta: 0.0510 | Regression_pt: 1.0195 | Regression_sin_phi: 0.0515 | LR: 2.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7218s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0802 | max_tensorboard_logging: 0.2063 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2971.6 | uss_MB: 1729.5 | child_rss_MB: 22770.0 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:31:53,559] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 06:31:53,560] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:34,863] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 06:32:34,863] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:40,241] INFO : Step 72000/500000 rank0 | Train Loss: 5.3242 | Train Loss Unweighted: 2.9626 | Train Loss Components: Classification: 0.0244 | Classification_binary: 2.7462 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0888 | Regression_eta: 0.0064 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2685 | Classification_binary: 2.7462 | Regression_cos_phi: 0.0584 | Regression_energy: 0.9561 | Regression_eta: 0.0506 | Regression_pt: 1.1968 | Regression_sin_phi: 0.0475 | LR: 2.45e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7297s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0268 | max_optimizer_step: 0.0836 | max_tensorboard_logging: 0.2082 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2971.4 | uss_MB: 1729.3 | child_rss_MB: 22771.1 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:32:40,243] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 06:32:40,293] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 06:33:17,131] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 06:33:17,131] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:33:54,796] INFO : Step 73000/500000 rank0 | Train Loss: 5.4729 | Train Loss Unweighted: 3.0059 | Train Loss Components: Classification: 0.0256 | Classification_binary: 2.7792 | Regression_cos_phi: 0.0092 | Regression_energy: 0.0914 | Regression_eta: 0.0066 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2810 | Classification_binary: 2.7792 | Regression_cos_phi: 0.1005 | Regression_energy: 0.9835 | Regression_eta: 0.0524 | Regression_pt: 1.2160 | Regression_sin_phi: 0.0603 | LR: 2.50e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7216s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0831 | max_tensorboard_logging: 0.2056 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2971.4 | uss_MB: 1729.3 | child_rss_MB: 22770.4 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:33:59,206] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 06:33:59,206] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:40,517] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 06:34:40,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:08,965] INFO : Step 74000/500000 rank0 | Train Loss: 5.4256 | Train Loss Unweighted: 2.9539 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7275 | Regression_cos_phi: 0.0092 | Regression_energy: 0.0910 | Regression_eta: 0.0052 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2341 | Classification_binary: 2.7275 | Regression_cos_phi: 0.1002 | Regression_energy: 0.9794 | Regression_eta: 0.0410 | Regression_pt: 1.2425 | Regression_sin_phi: 0.1008 | LR: 2.55e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7185s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0804 | max_tensorboard_logging: 0.2099 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2864.8 | uss_MB: 1622.7 | child_rss_MB: 22770.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:35:08,967] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 06:35:09,017] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 06:35:22,603] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 06:35:22,603] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:04,033] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 06:36:04,034] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:23,852] INFO : Step 75000/500000 rank0 | Train Loss: 5.3375 | Train Loss Unweighted: 3.0206 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.8072 | Regression_cos_phi: 0.0059 | Regression_energy: 0.0836 | Regression_eta: 0.0088 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2629 | Classification_binary: 2.8072 | Regression_cos_phi: 0.0645 | Regression_energy: 0.9000 | Regression_eta: 0.0697 | Regression_pt: 1.1577 | Regression_sin_phi: 0.0755 | LR: 2.60e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7412s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0302 | max_loss: 0.0021 | max_backward: 0.0270 | max_optimizer_step: 0.0826 | max_tensorboard_logging: 0.2077 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2870.6 | uss_MB: 1628.5 | child_rss_MB: 22769.7 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:36:46,436] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 06:36:46,436] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:27,172] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 06:37:27,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:37,228] INFO : Step 76000/500000 rank0 | Train Loss: 4.9390 | Train Loss Unweighted: 2.6609 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.4517 | Regression_cos_phi: 0.0063 | Regression_energy: 0.0839 | Regression_eta: 0.0072 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2405 | Classification_binary: 2.4517 | Regression_cos_phi: 0.0689 | Regression_energy: 0.9030 | Regression_eta: 0.0574 | Regression_pt: 1.1536 | Regression_sin_phi: 0.0638 | LR: 2.65e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7059s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0257 | max_optimizer_step: 0.0789 | max_tensorboard_logging: 0.2050 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2870.8 | uss_MB: 1628.7 | child_rss_MB: 22771.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:37:37,230] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 06:37:37,278] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 06:38:08,255] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 06:38:08,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:49,294] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 06:38:49,294] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:50,420] INFO : Step 77000/500000 rank0 | Train Loss: 4.3600 | Train Loss Unweighted: 2.4897 | Train Loss Components: Classification: 0.0249 | Classification_binary: 2.3173 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0681 | Regression_eta: 0.0060 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2734 | Classification_binary: 2.3173 | Regression_cos_phi: 0.0480 | Regression_energy: 0.7330 | Regression_eta: 0.0473 | Regression_pt: 0.9082 | Regression_sin_phi: 0.0328 | LR: 2.70e-04 | DataLoad Time: 0.0003s | Model Forward Time: 0.7517s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0022 | max_backward: 0.0266 | max_optimizer_step: 0.1088 | max_tensorboard_logging: 0.2081 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2871.0 | uss_MB: 1628.9 | child_rss_MB: 22773.2 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:39:31,606] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 06:39:31,606] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:04,709] INFO : Step 78000/500000 rank0 | Train Loss: 4.2693 | Train Loss Unweighted: 2.4605 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.2943 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0663 | Regression_eta: 0.0049 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2051 | Classification_binary: 2.2943 | Regression_cos_phi: 0.0636 | Regression_energy: 0.7130 | Regression_eta: 0.0389 | Regression_pt: 0.8962 | Regression_sin_phi: 0.0582 | LR: 2.75e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7212s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0022 | max_backward: 0.0268 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2071 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2871.0 | uss_MB: 1628.9 | child_rss_MB: 22772.1 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:40:04,711] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 06:40:04,759] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 06:40:13,396] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 06:40:13,396] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:54,619] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 06:40:54,620] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:41:18,752] INFO : Step 79000/500000 rank0 | Train Loss: 4.4165 | Train Loss Unweighted: 2.5798 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.4109 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0674 | Regression_eta: 0.0048 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2293 | Classification_binary: 2.4109 | Regression_cos_phi: 0.0548 | Regression_energy: 0.7252 | Regression_eta: 0.0383 | Regression_pt: 0.9012 | Regression_sin_phi: 0.0569 | LR: 2.80e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7303s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0235 | max_loss: 0.0021 | max_backward: 0.0327 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.2076 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2870.7 | uss_MB: 1628.6 | child_rss_MB: 22771.6 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:41:36,546] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 06:41:36,546] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:17,933] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 06:42:17,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:33,387] INFO : Step 80000/500000 rank0 | Train Loss: 4.7619 | Train Loss Unweighted: 2.6209 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.4237 | Regression_cos_phi: 0.0057 | Regression_energy: 0.0805 | Regression_eta: 0.0069 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 10.9849 | Classification_binary: 1.0000 | Regression_cos_phi: 10.9317 | Regression_energy: 10.7610 | Regression_eta: 7.9250 | Regression_pt: 13.7565 | Regression_sin_phi: 10.7636 | Task Weighted Losses: Classification: 0.2359 | Classification_binary: 2.4237 | Regression_cos_phi: 0.0626 | Regression_energy: 0.8662 | Regression_eta: 0.0546 | Regression_pt: 1.0558 | Regression_sin_phi: 0.0631 | LR: 2.85e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7247s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0022 | max_backward: 0.0336 | max_optimizer_step: 0.0811 | max_tensorboard_logging: 0.2098 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2871.4 | uss_MB: 1629.3 | child_rss_MB: 22772.5 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:42:33,389] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 06:42:33,435] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 06:42:33,436] INFO : Running validation on rank0 +rank0 [2026-11-14 06:42:33,436] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:42:33,436] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:42:33,543] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 06:42:33,624] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:42:33,701] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:42:33,749] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:42:33,795] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:42:33,840] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:42:33,886] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:42:33,931] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:42:33,999] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:42:34,077] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:42:34,154] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:42:34,200] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:42:34,246] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:42:34,291] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:42:34,337] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:42:34,383] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:42:34,449] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:42:34,454] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:42:34,455] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:42:34,457] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:42:34,458] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:42:34,460] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:42:34,461] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:42:34,463] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:42:34,464] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:42:34,466] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:42:34,468] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:42:34,469] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:42:34,471] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:42:34,472] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:42:34,474] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:42:34,475] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:42:35,408] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.7619 | Valid Loss=10.7437 | Train Loss Unweighted=2.6209 | Valid Loss Unweighted=4.3890 | Stale=3 | ETA=522.0m +rank0 [2026-11-14 06:42:35,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:42:35,503] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:42:35,512] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:42:35,560] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:42:35,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:42:36,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:42:36,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:42:36,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:42:36,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:42:36,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:42:36,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:42:37,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:42:37,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:42:37,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:42:37,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:42:37,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:42:37,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:42:38,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:42:38,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:42:38,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:42:38,491] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:42:38,498] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:42:38,892] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_10mm_seed12345_20261114_050301_075935/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:42:39,136] INFO : Plotted number of elements +rank0 [2026-11-14 06:42:40,054] INFO : Plotted jets +rank0 [2026-11-14 06:42:40,746] INFO : Plotted jet ratio +rank0 [2026-11-14 06:42:41,227] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 06:42:41,227] INFO : Training completed. Total time on device 0: 99.520min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..39afb8ef252d5409908368bdb9aaaf395f025d95 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:837ff1d621d7f6e717a1396ff8d1955534b769119765d10f52bbfdff920e20c0 +size 47063573